nicorepo 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +24 -0
- data/.nicorepo.yaml.sample +7 -0
- data/.rspec +1 -0
- data/Gemfile +5 -0
- data/LICENSE.txt +22 -0
- data/README.md +109 -0
- data/Rakefile +2 -0
- data/bin/nicorepo +6 -0
- data/lib/nicorepo/cli/cli.rb +154 -0
- data/lib/nicorepo/cli/config.rb +45 -0
- data/lib/nicorepo/log.rb +53 -0
- data/lib/nicorepo/version.rb +3 -0
- data/lib/nicorepo.rb +88 -0
- data/nicorepo.gemspec +31 -0
- data/spec/cli/cli_config_spec.rb +71 -0
- data/spec/cli/cli_spec.rb +85 -0
- data/spec/nicorepo_spec.rb +119 -0
- data/spec/spec_helper.rb +17 -0
- metadata +165 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 287151c7de9632af7c207fb45f87b9f2eba205f6
|
4
|
+
data.tar.gz: 6768a4c68e512fc8e9b6a984a47bd8eadf4d04ad
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: fcb8600bbc7a0237a19d9fd58a7133e50a235065a51dda0e7e4a7bcf1af98ba87d67237d5bfd4ba0b1221d16ebd91516dfa69c089b4eacc825cdb19508ed3ead
|
7
|
+
data.tar.gz: 1fa75e08fc371cdf417b58a6c068141bf87d8f9d7ccee547f040f294bbab87b7fe59966f0490e0c7d327227e0a14d62204e5043a9231c69500e8ba74dd4a4f4d
|
data/.gitignore
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
.bundle
|
4
|
+
.config
|
5
|
+
.yardoc
|
6
|
+
Gemfile.lock
|
7
|
+
InstalledFiles
|
8
|
+
_yardoc
|
9
|
+
coverage
|
10
|
+
doc/
|
11
|
+
lib/bundler/man
|
12
|
+
pkg
|
13
|
+
rdoc
|
14
|
+
spec/reports
|
15
|
+
test/tmp
|
16
|
+
test/version_tmp
|
17
|
+
tmp
|
18
|
+
*.bundle
|
19
|
+
*.so
|
20
|
+
*.o
|
21
|
+
*.a
|
22
|
+
mkmf.log
|
23
|
+
spec/resource/account.txt
|
24
|
+
|
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 upinetree
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,109 @@
|
|
1
|
+
# Nicorepo
|
2
|
+
ニコニコ動画のニコレポをスクレイピングするためのライブラリとスクリプトです。
|
3
|
+
|
4
|
+
- 欲しいニコレポログを、欲しい数だけ取得できます(例:投稿動画を30個)
|
5
|
+
- 複数ページにわたってログを取得できます
|
6
|
+
- スクリプトを使ってターミナル上からニコレポ確認したり、ブラウザにURL送ったりできます
|
7
|
+
|
8
|
+
|
9
|
+
# 環境とか
|
10
|
+
|
11
|
+
- ruby 2.0.0p247
|
12
|
+
- 依存Gem: `mechanize`, `launchy`
|
13
|
+
|
14
|
+
|
15
|
+
# 使い方
|
16
|
+
|
17
|
+
## スクリプト
|
18
|
+
|
19
|
+
bin/nicorepoを呼び出します。本体はlib/nicorepo/cli.rbです。
|
20
|
+
|
21
|
+
$ nicorepo command params
|
22
|
+
|
23
|
+
なお、起動前にアカウントを設定する必要があります。
|
24
|
+
|
25
|
+
config.yamlをnicorepoフォルダに作成して、下記のように記述してください。
|
26
|
+
|
27
|
+
mail: your@mail
|
28
|
+
pass: your_password
|
29
|
+
|
30
|
+
|
31
|
+
### command
|
32
|
+
|
33
|
+
all, a [disp_num] : すべてのニコレポを表示
|
34
|
+
videos, v [disp_num] [nest] : 投稿動画のみ
|
35
|
+
lives, l [disp_num] [nest] : 生放送のみ
|
36
|
+
interactive, i : 対話モード(後述)
|
37
|
+
|
38
|
+
- [disp\_num]の数だけニコレポを表示します。省略すると10個表示します
|
39
|
+
|
40
|
+
- [nest]は探しに行くページの最大数です。[nest]以上探しても[disp\_num]の数だけ見つけられない場合は諦めます。省略すると3ページです
|
41
|
+
|
42
|
+
- それぞれエイリアスが設定されています。all なら a だけで認識します
|
43
|
+
|
44
|
+
|
45
|
+
### 対話モード
|
46
|
+
|
47
|
+
`nicorepo i`で起動します。
|
48
|
+
対話モードは上記コマンドに加えて、下記の対話用コマンドが使用できます。
|
49
|
+
|
50
|
+
open, o [log_num] : 指定のニコレポログの対象URLをブラウザで開く
|
51
|
+
login : 再ログイン
|
52
|
+
exit : 対話モード終了
|
53
|
+
|
54
|
+
`all`や`videos`でニコレポを表示すると、各ログの最初に連番が振られます。
|
55
|
+
その連番を`open`に指定すると、対象のURLをブラウザで開きます。
|
56
|
+
|
57
|
+
|
58
|
+
### config.yaml
|
59
|
+
|
60
|
+
config.yamlでは、アカウントの設定の他に、取得するログの数、探しに行くページの数をコマンドごとに設定することが出来ます。
|
61
|
+
設定項目は以下のとおりです。
|
62
|
+
|
63
|
+
general: コマンド全体に定義した数を設定します
|
64
|
+
all, videos, lives: それぞれのコマンドに定義した数を設定します。generalより優先します
|
65
|
+
|
66
|
+
(例)
|
67
|
+
|
68
|
+
general:
|
69
|
+
num: 20
|
70
|
+
nest: 5
|
71
|
+
videos:
|
72
|
+
num: 5
|
73
|
+
nest: 10
|
74
|
+
|
75
|
+
|
76
|
+
## ライブラリ
|
77
|
+
|
78
|
+
ニコニコ動画へのログインは次のような感じです。
|
79
|
+
|
80
|
+
repo = Nicorepo.new
|
81
|
+
repo.login(mail, pass)
|
82
|
+
|
83
|
+
以降のニコレポ取得ですが、現在は3種類だけメソッドにしています。
|
84
|
+
|
85
|
+
all(req_num = LOGS_PER_PAGE)
|
86
|
+
videos(req_num = 3, page_nest_max = 5)
|
87
|
+
lives(req_num = 3, page_nest_max = 5)
|
88
|
+
|
89
|
+
取得したニコレポは`Nicorepo::Log`のArrayで返ってきます。
|
90
|
+
中身は、
|
91
|
+
|
92
|
+
@body # 本文。「〜さんが…しました」
|
93
|
+
@title # ログ対象の名前。動画名や生放送名など
|
94
|
+
@url # ログ対象のURL。動画URLや生放送URLなど
|
95
|
+
@author # ログ発生元ユーザ
|
96
|
+
@kind # ログの種類。CSSクラス名から抜粋したもの
|
97
|
+
@date # ログ発生日時
|
98
|
+
|
99
|
+
filtered_byを使うと、@kindを指定の条件でフィルタしてログを取得します。
|
100
|
+
|
101
|
+
filtered_by(filter, req_num = LOGS_PER_PAGE, page_nest_max = 1)
|
102
|
+
|
103
|
+
例えば:
|
104
|
+
|
105
|
+
logs = filtered_by('clip')
|
106
|
+
logs = filtered_by('seiga')
|
107
|
+
logs = filtered_by('mylist')
|
108
|
+
|
109
|
+
のような感じです。
|
data/Rakefile
ADDED
data/bin/nicorepo
ADDED
@@ -0,0 +1,154 @@
|
|
1
|
+
require 'launchy'
|
2
|
+
require 'readline'
|
3
|
+
require 'netrc'
|
4
|
+
|
5
|
+
class Nicorepo
|
6
|
+
class Cli
|
7
|
+
|
8
|
+
class LogExistenceError < StandardError; end
|
9
|
+
class LoginAccountError < StandardError; end
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
@repo = Nicorepo.new
|
13
|
+
@logs = nil
|
14
|
+
@conf = Nicorepo::Cli::Config.new
|
15
|
+
end
|
16
|
+
|
17
|
+
def run(argv)
|
18
|
+
cmd, num, nest = parse(argv)
|
19
|
+
help if cmd == 'help'
|
20
|
+
|
21
|
+
login
|
22
|
+
|
23
|
+
logs = exec_command(cmd, num, nest)
|
24
|
+
if logs
|
25
|
+
disp logs
|
26
|
+
else
|
27
|
+
case cmd
|
28
|
+
when 'interactive' then interactive_run
|
29
|
+
else help
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
# returns true when exit
|
35
|
+
def interactive_run
|
36
|
+
loop do
|
37
|
+
argv = Readline::readline("nicorepo > ", true).split
|
38
|
+
cmd, num, nest = parse(argv)
|
39
|
+
|
40
|
+
logs = exec_command(cmd, num, nest)
|
41
|
+
if logs
|
42
|
+
@logs = logs
|
43
|
+
disp @logs
|
44
|
+
else
|
45
|
+
case cmd
|
46
|
+
when 'open' then open_url(@logs, num)
|
47
|
+
when 'login' then login
|
48
|
+
when 'exit' then return true
|
49
|
+
else help_interactive; next
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
# options is now just for testing
|
56
|
+
def open_url(logs, num, options = {})
|
57
|
+
url = logs[num - 1].url
|
58
|
+
if url.nil?
|
59
|
+
puts "log existence error: please fetch logs"
|
60
|
+
raise LogExistenceError
|
61
|
+
end
|
62
|
+
|
63
|
+
Launchy.open(url, options) do |exception|
|
64
|
+
puts "Attempted to open #{url} and failed because #{exception}"
|
65
|
+
raise exception
|
66
|
+
end
|
67
|
+
|
68
|
+
return true
|
69
|
+
end
|
70
|
+
|
71
|
+
private
|
72
|
+
|
73
|
+
def parse(argv)
|
74
|
+
cmd = translate(argv.shift || 'help')
|
75
|
+
num = (argv.shift || @conf.num(cmd)).to_i
|
76
|
+
nest = (argv.shift || @conf.nest(cmd)).to_i
|
77
|
+
|
78
|
+
return cmd, num, nest
|
79
|
+
end
|
80
|
+
|
81
|
+
def login
|
82
|
+
n = Netrc.read
|
83
|
+
mail, pass = n["nicovideo.jp"]
|
84
|
+
raise LoginAccountError, "machine nicovideo.jp is not defined in .netrc" if mail.nil? || pass.nil?
|
85
|
+
|
86
|
+
begin
|
87
|
+
@repo.login(mail, pass)
|
88
|
+
rescue
|
89
|
+
raise LoginAccountError, "invalid mail or pass: mail = #{mail}"
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
# it returns
|
94
|
+
# - logs if succeed to exec exepcted command
|
95
|
+
# - nil if unexpected command given
|
96
|
+
def exec_command(cmd, num, nest)
|
97
|
+
logs = nil
|
98
|
+
|
99
|
+
case cmd
|
100
|
+
when 'all' then logs = @repo.all num
|
101
|
+
when 'videos' then logs = @repo.videos num, nest
|
102
|
+
when 'lives' then logs = @repo.lives num, nest
|
103
|
+
else return nil
|
104
|
+
end
|
105
|
+
|
106
|
+
return logs
|
107
|
+
end
|
108
|
+
|
109
|
+
ALIAS = {"a" => "all", "v" => "videos", "l" => "lives",
|
110
|
+
"o" => "open", "i" => "interactive"}
|
111
|
+
def translate(cmd)
|
112
|
+
if ALIAS.has_key?(cmd)
|
113
|
+
ALIAS[cmd]
|
114
|
+
else
|
115
|
+
cmd
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def help
|
120
|
+
puts ' usage: nicorepo command [params]'
|
121
|
+
puts ' command:'
|
122
|
+
help_commands
|
123
|
+
puts ' interactive, i - begin interactive mode'
|
124
|
+
exit 1
|
125
|
+
end
|
126
|
+
|
127
|
+
def help_interactive
|
128
|
+
puts ' usage: command [params]'
|
129
|
+
puts ' command:'
|
130
|
+
help_commands
|
131
|
+
puts ' open, o [log_num] - open url of given log number'
|
132
|
+
puts ' login'
|
133
|
+
puts ' exit'
|
134
|
+
end
|
135
|
+
|
136
|
+
def help_commands
|
137
|
+
puts <<-"EOS"
|
138
|
+
all, a [disp_num]
|
139
|
+
videos, v [disp_num] [nest]
|
140
|
+
lives, l [disp_num] [nest]
|
141
|
+
*disp_num - number of logs to display at once (default = 10)
|
142
|
+
*nest - max nesting level of pages to search (default = 3)
|
143
|
+
EOS
|
144
|
+
end
|
145
|
+
|
146
|
+
def disp(logs)
|
147
|
+
logs.each.with_index(1) do |log, i|
|
148
|
+
puts "[#{i}] #{log.body} on #{log.date.to_s}"
|
149
|
+
puts " '#{log.title}' (#{log.url})"
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
class Nicorepo
|
4
|
+
class Cli
|
5
|
+
class Config
|
6
|
+
|
7
|
+
class ReadError < StandardError; end
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
params = defaults.merge(load_config)
|
11
|
+
@nums = params["num"]
|
12
|
+
@nests = params["nest"]
|
13
|
+
end
|
14
|
+
|
15
|
+
def num(cmd)
|
16
|
+
@nums[cmd] || @nums["general"]
|
17
|
+
end
|
18
|
+
|
19
|
+
def nest(cmd)
|
20
|
+
@nests[cmd] || @nests["general"]
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def load_config
|
26
|
+
filename = File.expand_path('~/.nicorepo.yaml')
|
27
|
+
return {} unless File.exist?(filename)
|
28
|
+
|
29
|
+
open(filename) { |f| YAML.load(f.read) }
|
30
|
+
end
|
31
|
+
|
32
|
+
def defaults
|
33
|
+
{
|
34
|
+
"num" => {
|
35
|
+
"general" => 10
|
36
|
+
},
|
37
|
+
"nest" => {
|
38
|
+
"general" => 3
|
39
|
+
}
|
40
|
+
}
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
data/lib/nicorepo/log.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
class Nicorepo
|
2
|
+
|
3
|
+
class Log
|
4
|
+
attr_accessor :body, :title, :url, :author, :kind, :date
|
5
|
+
|
6
|
+
def initialize(node = nil)
|
7
|
+
if node.nil? then return end
|
8
|
+
@body = parse_body node
|
9
|
+
@title = parse_title node
|
10
|
+
@url = parse_url node
|
11
|
+
@author = parse_author node
|
12
|
+
@kind = parse_kind node
|
13
|
+
@date = parse_date node
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def parse_body(node)
|
19
|
+
node.search('div.log-body').first.inner_text.gsub(/(\t|\r|\n)/, "")
|
20
|
+
end
|
21
|
+
|
22
|
+
def parse_title(node)
|
23
|
+
node.search('div.log-target-info/a').first.inner_text
|
24
|
+
end
|
25
|
+
|
26
|
+
def parse_url(node)
|
27
|
+
node.search('div.log-target-info/a').first['href']
|
28
|
+
end
|
29
|
+
|
30
|
+
def parse_author(node)
|
31
|
+
node.search('div.log-body/a').first.inner_text
|
32
|
+
end
|
33
|
+
|
34
|
+
# example: 'log.log-community-video-upload' -> 'community-video-upload'
|
35
|
+
def parse_kind(node)
|
36
|
+
cls = node['class']
|
37
|
+
trim = 'log-'
|
38
|
+
|
39
|
+
index = cls.index(/#{trim}/)
|
40
|
+
return cls if index.nil?
|
41
|
+
|
42
|
+
index += trim.size
|
43
|
+
cls[index..cls.size]
|
44
|
+
end
|
45
|
+
|
46
|
+
def parse_date(node)
|
47
|
+
d = node.search('div.log-footer/div.log-footer-inner/a.log-footer-date/time').first['datetime']
|
48
|
+
Time.xmlschema(d).localtime
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
data/lib/nicorepo.rb
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'mechanize'
|
2
|
+
require 'nicorepo/log'
|
3
|
+
|
4
|
+
class Nicorepo
|
5
|
+
|
6
|
+
module URL
|
7
|
+
LOGIN = 'https://secure.nicovideo.jp/secure/login?site=niconico'
|
8
|
+
REPO_ALL = 'http://www.nicovideo.jp/my/top'
|
9
|
+
end
|
10
|
+
|
11
|
+
class LoginError < StandardError; end
|
12
|
+
class LogsAccessError < StandardError; end
|
13
|
+
|
14
|
+
LOGS_PER_PAGE = 20
|
15
|
+
|
16
|
+
attr_reader :agent
|
17
|
+
|
18
|
+
def initialize
|
19
|
+
@agent = Mechanize.new
|
20
|
+
@agent.ssl_version = 'SSLv3'
|
21
|
+
@agent.request_headers = { 'accept-language' => 'ja-JP', 'content-language' => 'ja-JP' }
|
22
|
+
end
|
23
|
+
|
24
|
+
def login(mail, pass)
|
25
|
+
page = @agent.post(URL::LOGIN, mail: mail, password: pass)
|
26
|
+
raise LoginError, "Failed to login" if page.header["x-niconico-authflag"] == '0'
|
27
|
+
end
|
28
|
+
|
29
|
+
def all(req_num = LOGS_PER_PAGE)
|
30
|
+
page_nest_max = req_num / LOGS_PER_PAGE + 1
|
31
|
+
fetch_logs(req_num, page_nest_max)
|
32
|
+
end
|
33
|
+
|
34
|
+
def videos(req_num = 3, page_nest_max = 5)
|
35
|
+
filtered_by('video-upload', req_num, page_nest_max)
|
36
|
+
end
|
37
|
+
|
38
|
+
def lives(req_num = 3, page_nest_max = 5)
|
39
|
+
filtered_by('live', req_num, page_nest_max)
|
40
|
+
end
|
41
|
+
|
42
|
+
def filtered_by(filter, req_num = LOGS_PER_PAGE, page_nest_max = 1)
|
43
|
+
fetch_logs(req_num, page_nest_max, filter)
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def fetch_logs(req_num, page_nest_max, filter = nil, url = URL::REPO_ALL)
|
49
|
+
return [] unless page_nest_max > 0
|
50
|
+
|
51
|
+
# fetch current logs
|
52
|
+
page = @agent.get(url)
|
53
|
+
begin
|
54
|
+
logs = log_nodes(page).map { |node| Log.new(node) }
|
55
|
+
rescue
|
56
|
+
raise LogsAccessError
|
57
|
+
end
|
58
|
+
logs.select!{ |log| log.kind =~ /#{filter}/ } if filter
|
59
|
+
|
60
|
+
if logs.size > req_num then
|
61
|
+
return logs[0, req_num]
|
62
|
+
end
|
63
|
+
|
64
|
+
# fetch next logs
|
65
|
+
if logs.size < req_num then
|
66
|
+
next_url = page.search('div.next-page/a').first['href']
|
67
|
+
begin
|
68
|
+
next_logs = fetch_logs(req_num - logs.size, page_nest_max - 1, filter, next_url)
|
69
|
+
rescue
|
70
|
+
warn '*** logs access error occurs ***'
|
71
|
+
return logs
|
72
|
+
else
|
73
|
+
logs += next_logs unless next_logs.nil?
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
return logs
|
78
|
+
end
|
79
|
+
|
80
|
+
def log_nodes(page)
|
81
|
+
page.search('div.timeline/div.log')
|
82
|
+
end
|
83
|
+
|
84
|
+
end
|
85
|
+
|
86
|
+
require "nicorepo/version"
|
87
|
+
require 'nicorepo/cli/cli'
|
88
|
+
require 'nicorepo/cli/config'
|
data/nicorepo.gemspec
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
lib = File.expand_path('../lib', __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require 'nicorepo/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "nicorepo"
|
7
|
+
spec.version = Nicorepo::VERSION
|
8
|
+
spec.authors = ["upinetree"]
|
9
|
+
spec.email = ["essequake@gmail.com"]
|
10
|
+
spec.summary = %q{Simple nicorepo scraper}
|
11
|
+
spec.description = %q{Scrapes and filters nicorepo reports from nicovideo.}
|
12
|
+
spec.homepage = "https://github.com/upinetree/nicorepo"
|
13
|
+
spec.license = "MIT"
|
14
|
+
|
15
|
+
spec.files = `git ls-files -z`.split("\x0")
|
16
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
|
+
spec.require_paths = ["lib"]
|
19
|
+
|
20
|
+
spec.required_ruby_version = ">= 2.0.0"
|
21
|
+
|
22
|
+
spec.add_dependency "mechanize", "~> 2.7"
|
23
|
+
spec.add_dependency "launchy", "~> 2.4"
|
24
|
+
spec.add_dependency "netrc", "~> 0.7"
|
25
|
+
|
26
|
+
spec.add_development_dependency "bundler", "~> 1.6"
|
27
|
+
spec.add_development_dependency "pry"
|
28
|
+
spec.add_development_dependency "rake"
|
29
|
+
spec.add_development_dependency "rspec"
|
30
|
+
end
|
31
|
+
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Nicorepo::Cli::Config do
|
4
|
+
let(:conf) { Nicorepo::Cli::Config.new }
|
5
|
+
|
6
|
+
before do
|
7
|
+
Nicorepo::Cli::Config.any_instance.stub(:load_config).and_return(config_values)
|
8
|
+
end
|
9
|
+
|
10
|
+
describe "#num" do
|
11
|
+
context "with 'all' command" do
|
12
|
+
context "'all' num is NOT defined" do
|
13
|
+
context "'general' num is NOT defined" do
|
14
|
+
let(:config_values) { {} }
|
15
|
+
|
16
|
+
it "should return default num" do
|
17
|
+
default_num = conf.send(:defaults)["num"]["general"]
|
18
|
+
conf.num("all").should eq(default_num)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
context "'general' num is defined" do
|
23
|
+
let(:general_num) { 20 }
|
24
|
+
let(:config_values) { { "num" => { "general" => general_num } } }
|
25
|
+
|
26
|
+
it "should return defined 'general' num" do
|
27
|
+
conf.num("all").should eq(general_num)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
context "'all' num is defined" do
|
33
|
+
context "'general' num is NOT defined" do
|
34
|
+
let(:all_num) { 20 }
|
35
|
+
let(:config_values) { { "num" => { "all" => all_num } } }
|
36
|
+
|
37
|
+
it "should return defined 'all' num" do
|
38
|
+
conf.num("all").should eq(all_num)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
context "'general' num is defined" do
|
43
|
+
let(:all_num) { 20 }
|
44
|
+
let(:general_num) { 15 }
|
45
|
+
let(:config_values) { { "num" => { "all" => all_num, "general" => general_num } } }
|
46
|
+
|
47
|
+
it "should return defined 'all' num" do
|
48
|
+
conf.num("all").should eq(all_num)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
describe "#nest" do
|
56
|
+
context "with 'all' command" do
|
57
|
+
context "'all' nest is defined" do
|
58
|
+
context "'general' nest is defined" do
|
59
|
+
let(:all_nest) { 10 }
|
60
|
+
let(:general_nest) { 5 }
|
61
|
+
let(:config_values) { { "nest" => { "all" => all_nest, "general" => general_nest } } }
|
62
|
+
|
63
|
+
it "should return defined 'all' nest" do
|
64
|
+
conf.nest("all").should eq(all_nest)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Nicorepo::Cli do
|
4
|
+
before(:all) do
|
5
|
+
@old_stderr = $stderr
|
6
|
+
$stderr = StringIO.new
|
7
|
+
end
|
8
|
+
|
9
|
+
describe "#run" do
|
10
|
+
|
11
|
+
let(:cli) { Nicorepo::Cli.new }
|
12
|
+
before(:each) { cli.stub(:configure) }
|
13
|
+
|
14
|
+
context "when login failed" do
|
15
|
+
it "should exit with error massage" do
|
16
|
+
begin
|
17
|
+
Nicorepo::Cli::Config.any_instance.stub(:account).and_return({mail: "invalid@mail.com", pass: "invalid_pass"})
|
18
|
+
cli.run([ 'i' ])
|
19
|
+
rescue SystemExit => se
|
20
|
+
se.status.should eq 1
|
21
|
+
$stderr.string.should match /invalid mail or pass/
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
context "when login succeed" do
|
27
|
+
|
28
|
+
before(:each) do
|
29
|
+
cli.stub(:login)
|
30
|
+
end
|
31
|
+
|
32
|
+
context "with command 'i'" do
|
33
|
+
it "should exec interactive mode" do
|
34
|
+
cli.should_receive(:interactive_run)
|
35
|
+
cli.run([ 'i' ])
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
context "with command 'lives'" do
|
40
|
+
it "should recieve logs of live" do
|
41
|
+
Nicorepo.any_instance.should_receive(:lives).and_return([Nicorepo::Log.new])
|
42
|
+
cli.run([ 'lives' ])
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
describe "#interactive_run" do
|
49
|
+
context "when entered 'exit'" do
|
50
|
+
it "should return true" do
|
51
|
+
cli = Nicorepo::Cli.new
|
52
|
+
Readline.stub!(:readline) { 'exit' }
|
53
|
+
cli.interactive_run.should be_true
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
describe "#open_url" do
|
59
|
+
context "with 1" do
|
60
|
+
before do
|
61
|
+
@cli = Nicorepo::Cli.new
|
62
|
+
@logs = [ Nicorepo::Log.new ]
|
63
|
+
end
|
64
|
+
|
65
|
+
it "should succeed to open url in browser with first log's url" do
|
66
|
+
@logs.first.url = 'http://www.nicovideo.jp'
|
67
|
+
expect{ @cli.open_url(@logs, 1, {dry_run: true}) }.to be_true
|
68
|
+
end
|
69
|
+
|
70
|
+
it "should raise error when url is nil" do
|
71
|
+
@logs.first.url = nil
|
72
|
+
expect{ @cli.open_url(@logs, 1, {dry_run: true}) }.to raise_error
|
73
|
+
end
|
74
|
+
|
75
|
+
it "should raise error when url is wrongs" do
|
76
|
+
@logs.first.url = 'hoge://piyo'
|
77
|
+
expect{ @cli.open_url(@logs, 1, {dry_run: true}) }.to raise_error
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
after(:all) do
|
83
|
+
$stderr = @old_stderr
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,119 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Nicorepo do
|
4
|
+
|
5
|
+
include Helper
|
6
|
+
|
7
|
+
before(:all) do
|
8
|
+
@nicorepo = Nicorepo.new
|
9
|
+
@account = right_account
|
10
|
+
|
11
|
+
@nicorepo.login(@account[:mail], @account[:pass])
|
12
|
+
end
|
13
|
+
|
14
|
+
describe "#login" do
|
15
|
+
context "with right account" do
|
16
|
+
it "should be success" do
|
17
|
+
@nicorepo.agent.should be_true
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
context "with wrong account" do
|
22
|
+
it "should raise error" do
|
23
|
+
repo = Nicorepo.new
|
24
|
+
lambda{ repo.login('test', 'testpass') }.should raise_error(Nicorepo::LoginError)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
describe "#all" do
|
30
|
+
context "without arguments" do
|
31
|
+
it "should return 20 logs" do
|
32
|
+
@nicorepo.all.should have(20).logs
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
context "with 5" do
|
37
|
+
it "should return 5 logs" do
|
38
|
+
@nicorepo.all(5).should have(5).logs
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
context "with 30" do
|
43
|
+
it "should return 30 logs" do
|
44
|
+
@nicorepo.all(30).should have(30).logs
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
context "with 50" do
|
49
|
+
it "should return 50 logs" do
|
50
|
+
@nicorepo.all(50).should have(50).logs
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
describe "#videos" do
|
56
|
+
context "without arguments" do
|
57
|
+
it "should return 3 logs at the most" do
|
58
|
+
@nicorepo.videos.should have_at_most(3).logs
|
59
|
+
end
|
60
|
+
|
61
|
+
it "should return only video logs" do
|
62
|
+
videos = @nicorepo.videos
|
63
|
+
except_video = videos.reject{ |v| v.kind =~ /video/ }
|
64
|
+
except_video.size.should eq 0
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
context "with req_num = 5, page_nest_max = 3" do
|
69
|
+
it "should return 5 logs at the most" do
|
70
|
+
@nicorepo.videos(5, 3).should have_at_most(5).logs
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
describe "#lives" do
|
76
|
+
it "should return only live logs" do
|
77
|
+
lives = @nicorepo.lives
|
78
|
+
except_live = lives.reject{ |l| l.kind =~ /live/ }
|
79
|
+
except_live.size.should eq 0
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
describe Nicorepo::Log do
|
84
|
+
before(:all) do
|
85
|
+
@log = @nicorepo.all.first
|
86
|
+
p @log
|
87
|
+
end
|
88
|
+
|
89
|
+
it "should have the body" do
|
90
|
+
@log.body.should be_true
|
91
|
+
end
|
92
|
+
|
93
|
+
it "should have the title" do
|
94
|
+
@log.title.should be_true
|
95
|
+
end
|
96
|
+
|
97
|
+
it "should have the url" do
|
98
|
+
@log.url.should be_true
|
99
|
+
end
|
100
|
+
|
101
|
+
it "should have the author" do
|
102
|
+
@log.author.should be_true
|
103
|
+
end
|
104
|
+
|
105
|
+
it "should have the log-kind" do
|
106
|
+
@log.kind.should be_true
|
107
|
+
end
|
108
|
+
|
109
|
+
it "should have the date" do
|
110
|
+
@log.date.should be_true
|
111
|
+
end
|
112
|
+
|
113
|
+
after(:all) do
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
after(:all) do
|
118
|
+
end
|
119
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,165 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: nicorepo
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.2
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- upinetree
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-04-29 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: mechanize
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2.7'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2.7'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: launchy
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '2.4'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '2.4'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: netrc
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0.7'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0.7'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: bundler
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.6'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.6'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pry
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rake
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rspec
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
description: Scrapes and filters nicorepo reports from nicovideo.
|
112
|
+
email:
|
113
|
+
- essequake@gmail.com
|
114
|
+
executables:
|
115
|
+
- nicorepo
|
116
|
+
extensions: []
|
117
|
+
extra_rdoc_files: []
|
118
|
+
files:
|
119
|
+
- ".gitignore"
|
120
|
+
- ".nicorepo.yaml.sample"
|
121
|
+
- ".rspec"
|
122
|
+
- Gemfile
|
123
|
+
- LICENSE.txt
|
124
|
+
- README.md
|
125
|
+
- Rakefile
|
126
|
+
- bin/nicorepo
|
127
|
+
- lib/nicorepo.rb
|
128
|
+
- lib/nicorepo/cli/cli.rb
|
129
|
+
- lib/nicorepo/cli/config.rb
|
130
|
+
- lib/nicorepo/log.rb
|
131
|
+
- lib/nicorepo/version.rb
|
132
|
+
- nicorepo.gemspec
|
133
|
+
- spec/cli/cli_config_spec.rb
|
134
|
+
- spec/cli/cli_spec.rb
|
135
|
+
- spec/nicorepo_spec.rb
|
136
|
+
- spec/spec_helper.rb
|
137
|
+
homepage: https://github.com/upinetree/nicorepo
|
138
|
+
licenses:
|
139
|
+
- MIT
|
140
|
+
metadata: {}
|
141
|
+
post_install_message:
|
142
|
+
rdoc_options: []
|
143
|
+
require_paths:
|
144
|
+
- lib
|
145
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
146
|
+
requirements:
|
147
|
+
- - ">="
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: 2.0.0
|
150
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
151
|
+
requirements:
|
152
|
+
- - ">="
|
153
|
+
- !ruby/object:Gem::Version
|
154
|
+
version: '0'
|
155
|
+
requirements: []
|
156
|
+
rubyforge_project:
|
157
|
+
rubygems_version: 2.2.2
|
158
|
+
signing_key:
|
159
|
+
specification_version: 4
|
160
|
+
summary: Simple nicorepo scraper
|
161
|
+
test_files:
|
162
|
+
- spec/cli/cli_config_spec.rb
|
163
|
+
- spec/cli/cli_spec.rb
|
164
|
+
- spec/nicorepo_spec.rb
|
165
|
+
- spec/spec_helper.rb
|