nicorepo 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +0 -1
- data/.nicorepo.yaml.sample +2 -2
- data/README.md +60 -75
- data/lib/nicorepo/cli/cli.rb +38 -39
- data/lib/nicorepo/cli/config.rb +8 -8
- data/lib/nicorepo/{log.rb → parser.rb} +34 -13
- data/lib/nicorepo/report.rb +15 -0
- data/lib/nicorepo/reports.rb +74 -0
- data/lib/nicorepo/version.rb +1 -1
- data/lib/nicorepo.rb +13 -60
- data/spec/cli/cli_config_spec.rb +31 -31
- data/spec/nicorepo_spec.rb +19 -77
- data/spec/spec_helper.rb +1 -14
- metadata +4 -4
- data/spec/cli/cli_spec.rb +0 -85
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4d0c8ac38ef205c58bc61e5014580cf41126433f
|
4
|
+
data.tar.gz: 64fe13068e72c8edae3d9feb8ddab9e3f5fcf1d1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 149c12f8221bee89fd75e8ae6ea344557df39c16a2e86c0aa4bf4c49872125a56b535907bf6e93e2dc8f58016c740b7b9e00ee65b5a30abec743aebc0ea3889e
|
7
|
+
data.tar.gz: 6e76250ea97bc1a16bba04c8323e911972d93edd814f4b35942fdea4de3c1a7f2d3f5cb8997709e2e006cee08aa8879435817c899476e0c25ff3378b13b1088d
|
data/.gitignore
CHANGED
data/.nicorepo.yaml.sample
CHANGED
data/README.md
CHANGED
@@ -1,109 +1,94 @@
|
|
1
1
|
# Nicorepo
|
2
|
-
ニコニコ動画のニコレポをスクレイピングするためのライブラリとスクリプトです。
|
3
2
|
|
4
|
-
|
5
|
-
- 複数ページにわたってログを取得できます
|
6
|
-
- スクリプトを使ってターミナル上からニコレポ確認したり、ブラウザにURL送ったりできます
|
3
|
+
Nicorepo is scraper and filter of nicorepo on nicovideo.
|
7
4
|
|
5
|
+
- filter reports by kind of them
|
6
|
+
- specify number and pages to fetch reports
|
7
|
+
- open url in browser
|
8
8
|
|
9
|
-
|
9
|
+
It requires ruby 2.0.0.
|
10
10
|
|
11
|
-
|
12
|
-
- 依存Gem: `mechanize`, `launchy`
|
11
|
+
## Installation
|
13
12
|
|
13
|
+
Add this line to your application's Gemfile:
|
14
14
|
|
15
|
-
|
15
|
+
gem 'nicorepo'
|
16
16
|
|
17
|
-
|
17
|
+
And then execute:
|
18
18
|
|
19
|
-
|
19
|
+
$ bundle
|
20
20
|
|
21
|
-
|
21
|
+
Or install it yourself as:
|
22
22
|
|
23
|
-
|
23
|
+
$ gem install nicorepo
|
24
24
|
|
25
|
-
|
25
|
+
## Usage
|
26
26
|
|
27
|
-
|
28
|
-
pass: your_password
|
27
|
+
### Authentication
|
29
28
|
|
29
|
+
Nicorepo supports reading netrc file.
|
30
30
|
|
31
|
-
|
31
|
+
Add following lines to your netrc file (`~/.netrc`)
|
32
32
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
33
|
+
```
|
34
|
+
machine nicovideo.jp
|
35
|
+
login your@email.account
|
36
|
+
password your-password
|
37
|
+
```
|
37
38
|
|
38
|
-
|
39
|
+
### Start nicorepo cli as interactive mode
|
39
40
|
|
40
|
-
|
41
|
+
$ nicorepo i
|
41
42
|
|
42
|
-
|
43
|
+
You can use following commands in interactive cli to fetch nicorepos.
|
44
|
+
For example, if you want 20 video reports by searcing over 5 pages, the command will be,
|
43
45
|
|
46
|
+
> video 20 5
|
44
47
|
|
45
|
-
|
48
|
+
Or you can also use aliases.
|
46
49
|
|
47
|
-
|
48
|
-
対話モードは上記コマンドに加えて、下記の対話用コマンドが使用できます。
|
50
|
+
> v 20 5
|
49
51
|
|
50
|
-
|
51
|
-
login : 再ログイン
|
52
|
-
exit : 対話モード終了
|
52
|
+
And each commad has default value so it is simply used like,
|
53
53
|
|
54
|
-
`
|
55
|
-
|
54
|
+
# it means `video 10 3`
|
55
|
+
> v
|
56
56
|
|
57
|
+
**Commands**
|
57
58
|
|
58
|
-
|
59
|
+
command | alias | params | description
|
60
|
+
---------|-------|------------------------|-------------------------------------
|
61
|
+
all | a | request_num | all reports
|
62
|
+
videos | v | request_num limit_page | only videos
|
63
|
+
lives | l | request_num limit_page | only lives
|
64
|
+
open | o | report_num | open the specified report url in the browser
|
65
|
+
login | | | re-login
|
66
|
+
exit | | | exit nicorepo
|
59
67
|
|
60
|
-
|
61
|
-
設定項目は以下のとおりです。
|
68
|
+
### Configuration
|
62
69
|
|
63
|
-
|
64
|
-
|
70
|
+
You can configure default `request_num` and `limit_page` by adding `~/.nicorepo.yaml` if you want.
|
71
|
+
Please refer the sample `nicorepo/.nicorepo.yaml.sample` or copy it to your home directory.
|
65
72
|
|
66
|
-
|
73
|
+
**Sample**
|
67
74
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
75
|
+
```
|
76
|
+
request_num:
|
77
|
+
general: 20
|
78
|
+
videos: 5
|
79
|
+
limit_page:
|
80
|
+
general: 5
|
81
|
+
videos: 10
|
82
|
+
```
|
74
83
|
|
84
|
+
- `general`: used in all commands
|
85
|
+
- `all`, `videos`, `lives`: used in each command, has priority than `general`
|
75
86
|
|
76
|
-
##
|
87
|
+
## Contributing
|
77
88
|
|
78
|
-
|
89
|
+
1. Fork it ( https://github.com/upinetree/nicorepo/fork )
|
90
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
91
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
92
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
93
|
+
5. Create a new Pull Request
|
79
94
|
|
80
|
-
repo = Nicorepo.new
|
81
|
-
repo.login(mail, pass)
|
82
|
-
|
83
|
-
以降のニコレポ取得ですが、現在は3種類だけメソッドにしています。
|
84
|
-
|
85
|
-
all(req_num = LOGS_PER_PAGE)
|
86
|
-
videos(req_num = 3, page_nest_max = 5)
|
87
|
-
lives(req_num = 3, page_nest_max = 5)
|
88
|
-
|
89
|
-
取得したニコレポは`Nicorepo::Log`のArrayで返ってきます。
|
90
|
-
中身は、
|
91
|
-
|
92
|
-
@body # 本文。「〜さんが…しました」
|
93
|
-
@title # ログ対象の名前。動画名や生放送名など
|
94
|
-
@url # ログ対象のURL。動画URLや生放送URLなど
|
95
|
-
@author # ログ発生元ユーザ
|
96
|
-
@kind # ログの種類。CSSクラス名から抜粋したもの
|
97
|
-
@date # ログ発生日時
|
98
|
-
|
99
|
-
filtered_byを使うと、@kindを指定の条件でフィルタしてログを取得します。
|
100
|
-
|
101
|
-
filtered_by(filter, req_num = LOGS_PER_PAGE, page_nest_max = 1)
|
102
|
-
|
103
|
-
例えば:
|
104
|
-
|
105
|
-
logs = filtered_by('clip')
|
106
|
-
logs = filtered_by('seiga')
|
107
|
-
logs = filtered_by('mylist')
|
108
|
-
|
109
|
-
のような感じです。
|
data/lib/nicorepo/cli/cli.rb
CHANGED
@@ -5,24 +5,24 @@ require 'netrc'
|
|
5
5
|
class Nicorepo
|
6
6
|
class Cli
|
7
7
|
|
8
|
-
class
|
8
|
+
class ReportExistenceError < StandardError; end
|
9
9
|
class LoginAccountError < StandardError; end
|
10
10
|
|
11
11
|
def initialize
|
12
12
|
@repo = Nicorepo.new
|
13
|
-
@
|
13
|
+
@reports = nil
|
14
14
|
@conf = Nicorepo::Cli::Config.new
|
15
15
|
end
|
16
16
|
|
17
17
|
def run(argv)
|
18
|
-
cmd,
|
18
|
+
cmd, request_num, limit_page = parse(argv)
|
19
19
|
help if cmd == 'help'
|
20
20
|
|
21
21
|
login
|
22
22
|
|
23
|
-
|
24
|
-
if
|
25
|
-
disp
|
23
|
+
reports = exec_command(cmd, request_num, limit_page)
|
24
|
+
if reports
|
25
|
+
disp reports
|
26
26
|
else
|
27
27
|
case cmd
|
28
28
|
when 'interactive' then interactive_run
|
@@ -35,15 +35,15 @@ class Nicorepo
|
|
35
35
|
def interactive_run
|
36
36
|
loop do
|
37
37
|
argv = Readline::readline("nicorepo > ", true).split
|
38
|
-
cmd,
|
38
|
+
cmd, request_num, limit_page = parse(argv)
|
39
39
|
|
40
|
-
|
41
|
-
if
|
42
|
-
@
|
43
|
-
disp @
|
40
|
+
reports = exec_command(cmd, request_num, limit_page)
|
41
|
+
if reports
|
42
|
+
@reports = reports
|
43
|
+
disp @reports
|
44
44
|
else
|
45
45
|
case cmd
|
46
|
-
when 'open' then open_url(@
|
46
|
+
when 'open' then open_url(@reports, request_num)
|
47
47
|
when 'login' then login
|
48
48
|
when 'exit' then return true
|
49
49
|
else help_interactive; next
|
@@ -53,11 +53,11 @@ class Nicorepo
|
|
53
53
|
end
|
54
54
|
|
55
55
|
# options is now just for testing
|
56
|
-
def open_url(
|
57
|
-
url =
|
56
|
+
def open_url(reports, request_num, options = {})
|
57
|
+
url = reports[request_num - 1].url
|
58
58
|
if url.nil?
|
59
|
-
puts "
|
60
|
-
raise
|
59
|
+
puts "report existence error: please fetch reports"
|
60
|
+
raise ReportExistenceError
|
61
61
|
end
|
62
62
|
|
63
63
|
Launchy.open(url, options) do |exception|
|
@@ -72,15 +72,14 @@ class Nicorepo
|
|
72
72
|
|
73
73
|
def parse(argv)
|
74
74
|
cmd = translate(argv.shift || 'help')
|
75
|
-
|
76
|
-
|
75
|
+
request_num = (argv.shift || @conf.request_num(cmd)).to_i
|
76
|
+
limit_page = (argv.shift || @conf.limit_page(cmd)).to_i
|
77
77
|
|
78
|
-
return cmd,
|
78
|
+
return cmd, request_num, limit_page
|
79
79
|
end
|
80
80
|
|
81
81
|
def login
|
82
|
-
|
83
|
-
mail, pass = n["nicovideo.jp"]
|
82
|
+
mail, pass = Netrc.read["nicovideo.jp"]
|
84
83
|
raise LoginAccountError, "machine nicovideo.jp is not defined in .netrc" if mail.nil? || pass.nil?
|
85
84
|
|
86
85
|
begin
|
@@ -91,19 +90,19 @@ class Nicorepo
|
|
91
90
|
end
|
92
91
|
|
93
92
|
# it returns
|
94
|
-
# -
|
95
|
-
# - nil
|
96
|
-
def exec_command(cmd,
|
97
|
-
|
93
|
+
# - reports if succeed to exec exepcted command
|
94
|
+
# - nil if unexpected command given
|
95
|
+
def exec_command(cmd, request_num, limit_page)
|
96
|
+
reports = nil
|
98
97
|
|
99
98
|
case cmd
|
100
|
-
when 'all' then
|
101
|
-
when 'videos' then
|
102
|
-
when 'lives' then
|
99
|
+
when 'all' then reports = @repo.all request_num
|
100
|
+
when 'videos' then reports = @repo.videos request_num, limit_page
|
101
|
+
when 'lives' then reports = @repo.lives request_num, limit_page
|
103
102
|
else return nil
|
104
103
|
end
|
105
104
|
|
106
|
-
return
|
105
|
+
return reports
|
107
106
|
end
|
108
107
|
|
109
108
|
ALIAS = {"a" => "all", "v" => "videos", "l" => "lives",
|
@@ -128,25 +127,25 @@ class Nicorepo
|
|
128
127
|
puts ' usage: command [params]'
|
129
128
|
puts ' command:'
|
130
129
|
help_commands
|
131
|
-
puts ' open, o [
|
130
|
+
puts ' open, o [report_num] - open url of given report number'
|
132
131
|
puts ' login'
|
133
132
|
puts ' exit'
|
134
133
|
end
|
135
134
|
|
136
135
|
def help_commands
|
137
136
|
puts <<-"EOS"
|
138
|
-
all, a [
|
139
|
-
videos, v [
|
140
|
-
lives, l [
|
141
|
-
*
|
142
|
-
*
|
137
|
+
all, a [request_num]
|
138
|
+
videos, v [request_num] [limit_page]
|
139
|
+
lives, l [request_num] [limit_page]
|
140
|
+
*request_num - number of reports to fetch from nicovideo (default = 10)
|
141
|
+
*limit_page - limit page to fetch reports(default = 3)
|
143
142
|
EOS
|
144
143
|
end
|
145
144
|
|
146
|
-
def disp(
|
147
|
-
|
148
|
-
puts "[#{i}] #{
|
149
|
-
puts " '#{
|
145
|
+
def disp(reports)
|
146
|
+
reports.each.with_index(1) do |report, i|
|
147
|
+
puts "[#{i}] #{report.body} on #{report.date.to_s}"
|
148
|
+
puts " '#{report.title}' (#{report.url})"
|
150
149
|
end
|
151
150
|
end
|
152
151
|
end
|
data/lib/nicorepo/cli/config.rb
CHANGED
@@ -8,16 +8,16 @@ class Nicorepo
|
|
8
8
|
|
9
9
|
def initialize
|
10
10
|
params = defaults.merge(load_config)
|
11
|
-
@
|
12
|
-
@
|
11
|
+
@request_nums = params["request_num"]
|
12
|
+
@limit_pages = params["limit_page"]
|
13
13
|
end
|
14
14
|
|
15
|
-
def
|
16
|
-
@
|
15
|
+
def request_num(cmd)
|
16
|
+
@request_nums[cmd] || @request_nums["general"]
|
17
17
|
end
|
18
18
|
|
19
|
-
def
|
20
|
-
@
|
19
|
+
def limit_page(cmd)
|
20
|
+
@limit_pages[cmd] || @limit_pages["general"]
|
21
21
|
end
|
22
22
|
|
23
23
|
private
|
@@ -31,10 +31,10 @@ class Nicorepo
|
|
31
31
|
|
32
32
|
def defaults
|
33
33
|
{
|
34
|
-
"
|
34
|
+
"request_num" => {
|
35
35
|
"general" => 10
|
36
36
|
},
|
37
|
-
"
|
37
|
+
"limit_page" => {
|
38
38
|
"general" => 3
|
39
39
|
}
|
40
40
|
}
|
@@ -1,20 +1,43 @@
|
|
1
1
|
class Nicorepo
|
2
|
+
class Parser
|
3
|
+
def initialize(agent)
|
4
|
+
@agent = agent
|
5
|
+
end
|
2
6
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
@title = parse_title node
|
10
|
-
@url = parse_url node
|
11
|
-
@author = parse_author node
|
12
|
-
@kind = parse_kind node
|
13
|
-
@date = parse_date node
|
7
|
+
def parse_page(url)
|
8
|
+
page = @agent.get(url)
|
9
|
+
{
|
10
|
+
reports_attrs: parse_reports(page),
|
11
|
+
next_url: next_url(page)
|
12
|
+
}
|
14
13
|
end
|
15
14
|
|
16
15
|
private
|
17
16
|
|
17
|
+
def parse_reports(page)
|
18
|
+
nodes = report_nodes(page)
|
19
|
+
nodes.map { |node| report_attrs(node) }
|
20
|
+
end
|
21
|
+
|
22
|
+
def report_attrs(node)
|
23
|
+
{
|
24
|
+
body: parse_body(node),
|
25
|
+
title: parse_title(node),
|
26
|
+
url: parse_url(node),
|
27
|
+
author: parse_author(node),
|
28
|
+
kind: parse_kind(node),
|
29
|
+
date: parse_date(node)
|
30
|
+
}
|
31
|
+
end
|
32
|
+
|
33
|
+
def report_nodes(page)
|
34
|
+
page.search('div.timeline/div.log')
|
35
|
+
end
|
36
|
+
|
37
|
+
def next_url(page)
|
38
|
+
page.search('a.next-page-link').first['href']
|
39
|
+
end
|
40
|
+
|
18
41
|
def parse_body(node)
|
19
42
|
node.search('div.log-body').first.inner_text.gsub(/(\t|\r|\n)/, "")
|
20
43
|
end
|
@@ -47,7 +70,5 @@ class Nicorepo
|
|
47
70
|
d = node.search('div.log-footer/div.log-footer-inner/a.log-footer-date/time').first['datetime']
|
48
71
|
Time.xmlschema(d).localtime
|
49
72
|
end
|
50
|
-
|
51
73
|
end
|
52
|
-
|
53
74
|
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class Nicorepo
|
2
|
+
class Report
|
3
|
+
attr_accessor :body, :title, :url, :author, :kind, :date
|
4
|
+
|
5
|
+
def initialize(attrs)
|
6
|
+
@body = attrs[:body]
|
7
|
+
@title = attrs[:title]
|
8
|
+
@url = attrs[:url]
|
9
|
+
@author = attrs[:author]
|
10
|
+
@kind = attrs[:kind]
|
11
|
+
@date = attrs[:date]
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'nicorepo/report'
|
2
|
+
require 'nicorepo/parser'
|
3
|
+
require 'forwardable'
|
4
|
+
|
5
|
+
class Nicorepo
|
6
|
+
class Reports
|
7
|
+
extend Forwardable
|
8
|
+
|
9
|
+
class ReportsAccessError < StandardError; end
|
10
|
+
|
11
|
+
TOP_URL = 'http://www.nicovideo.jp/my/top'
|
12
|
+
|
13
|
+
attr_reader :reports
|
14
|
+
def_delegators :@reports, :size
|
15
|
+
|
16
|
+
def initialize(parser)
|
17
|
+
@parser = parser
|
18
|
+
@reports = []
|
19
|
+
end
|
20
|
+
|
21
|
+
def fetch(request_num, limit_page)
|
22
|
+
@reports = fetch_recursively(request_num, limit_page)
|
23
|
+
end
|
24
|
+
|
25
|
+
def fetch_with_filtere(filter, request_num, limit_page)
|
26
|
+
@reports = fetch_recursively(request_num, limit_page, filter)
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def fetch_recursively(request_num, limit_page, filter = nil, url = TOP_URL)
|
32
|
+
return [] unless limit_page > 0
|
33
|
+
|
34
|
+
# fetch current reports
|
35
|
+
page = @parser.parse_page(url)
|
36
|
+
begin
|
37
|
+
reports = page[:reports_attrs].map { |attrs| Report.new(attrs) }
|
38
|
+
rescue
|
39
|
+
raise ReportsAccessError
|
40
|
+
end
|
41
|
+
reports.select!{ |report| report.kind =~ /#{filter}/ } if filter
|
42
|
+
|
43
|
+
if reports.size > request_num then
|
44
|
+
return reports[0, request_num]
|
45
|
+
end
|
46
|
+
|
47
|
+
# recursively fetch next reports
|
48
|
+
if reports.size < request_num then
|
49
|
+
begin
|
50
|
+
next_reports = fetch_recursively(request_num - reports.size, limit_page - 1, filter, page[:next_url])
|
51
|
+
rescue
|
52
|
+
return reports
|
53
|
+
else
|
54
|
+
reports += next_reports unless next_reports.nil?
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
return reports
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
class VideoReports < Reports
|
63
|
+
def fetch(request_num, limit_page)
|
64
|
+
fetch_with_filtere('video-upload', request_num, limit_page)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
class LiveReports < Reports
|
69
|
+
def fetch(request_num, limit_page)
|
70
|
+
fetch_with_filtere('live', request_num, limit_page)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
data/lib/nicorepo/version.rb
CHANGED
data/lib/nicorepo.rb
CHANGED
@@ -1,17 +1,11 @@
|
|
1
1
|
require 'mechanize'
|
2
|
-
require 'nicorepo/
|
2
|
+
require 'nicorepo/reports'
|
3
3
|
|
4
4
|
class Nicorepo
|
5
|
-
|
6
|
-
module URL
|
7
|
-
LOGIN = 'https://secure.nicovideo.jp/secure/login?site=niconico'
|
8
|
-
REPO_ALL = 'http://www.nicovideo.jp/my/top'
|
9
|
-
end
|
10
|
-
|
11
5
|
class LoginError < StandardError; end
|
12
|
-
class LogsAccessError < StandardError; end
|
13
6
|
|
14
|
-
|
7
|
+
PER_PAGE = 20
|
8
|
+
LOGIN_URL = 'https://secure.nicovideo.jp/secure/login?site=niconico'
|
15
9
|
|
16
10
|
attr_reader :agent
|
17
11
|
|
@@ -19,70 +13,29 @@ class Nicorepo
|
|
19
13
|
@agent = Mechanize.new
|
20
14
|
@agent.ssl_version = 'SSLv3'
|
21
15
|
@agent.request_headers = { 'accept-language' => 'ja-JP', 'content-language' => 'ja-JP' }
|
16
|
+
@parser = Parser.new(@agent)
|
22
17
|
end
|
23
18
|
|
24
19
|
def login(mail, pass)
|
25
|
-
page = @agent.post(
|
20
|
+
page = @agent.post(LOGIN_URL, mail: mail, password: pass)
|
26
21
|
raise LoginError, "Failed to login" if page.header["x-niconico-authflag"] == '0'
|
27
22
|
end
|
28
23
|
|
29
|
-
def all(
|
30
|
-
|
31
|
-
|
32
|
-
end
|
33
|
-
|
34
|
-
def videos(req_num = 3, page_nest_max = 5)
|
35
|
-
filtered_by('video-upload', req_num, page_nest_max)
|
36
|
-
end
|
37
|
-
|
38
|
-
def lives(req_num = 3, page_nest_max = 5)
|
39
|
-
filtered_by('live', req_num, page_nest_max)
|
40
|
-
end
|
41
|
-
|
42
|
-
def filtered_by(filter, req_num = LOGS_PER_PAGE, page_nest_max = 1)
|
43
|
-
fetch_logs(req_num, page_nest_max, filter)
|
24
|
+
def all(request_num = PER_PAGE)
|
25
|
+
limit_page = request_num / PER_PAGE + 1
|
26
|
+
Reports.new(@parser).fetch(request_num, limit_page)
|
44
27
|
end
|
45
28
|
|
46
|
-
|
47
|
-
|
48
|
-
def fetch_logs(req_num, page_nest_max, filter = nil, url = URL::REPO_ALL)
|
49
|
-
return [] unless page_nest_max > 0
|
50
|
-
|
51
|
-
# fetch current logs
|
52
|
-
page = @agent.get(url)
|
53
|
-
begin
|
54
|
-
logs = log_nodes(page).map { |node| Log.new(node) }
|
55
|
-
rescue
|
56
|
-
raise LogsAccessError
|
57
|
-
end
|
58
|
-
logs.select!{ |log| log.kind =~ /#{filter}/ } if filter
|
59
|
-
|
60
|
-
if logs.size > req_num then
|
61
|
-
return logs[0, req_num]
|
62
|
-
end
|
63
|
-
|
64
|
-
# fetch next logs
|
65
|
-
if logs.size < req_num then
|
66
|
-
next_url = page.search('div.next-page/a').first['href']
|
67
|
-
begin
|
68
|
-
next_logs = fetch_logs(req_num - logs.size, page_nest_max - 1, filter, next_url)
|
69
|
-
rescue
|
70
|
-
warn '*** logs access error occurs ***'
|
71
|
-
return logs
|
72
|
-
else
|
73
|
-
logs += next_logs unless next_logs.nil?
|
74
|
-
end
|
75
|
-
end
|
76
|
-
|
77
|
-
return logs
|
29
|
+
def videos(request_num = 3, limit_page = 5)
|
30
|
+
VideoReports.new(@parser).fetch(request_num, limit_page)
|
78
31
|
end
|
79
32
|
|
80
|
-
def
|
81
|
-
|
33
|
+
def lives(request_num = 3, limit_page = 5)
|
34
|
+
LiveReports.new(@parser).fetch(request_num, limit_page)
|
82
35
|
end
|
83
|
-
|
84
36
|
end
|
85
37
|
|
86
38
|
require "nicorepo/version"
|
87
39
|
require 'nicorepo/cli/cli'
|
88
40
|
require 'nicorepo/cli/config'
|
41
|
+
|
data/spec/cli/cli_config_spec.rb
CHANGED
@@ -7,61 +7,61 @@ describe Nicorepo::Cli::Config do
|
|
7
7
|
Nicorepo::Cli::Config.any_instance.stub(:load_config).and_return(config_values)
|
8
8
|
end
|
9
9
|
|
10
|
-
describe "#
|
10
|
+
describe "#request_num" do
|
11
11
|
context "with 'all' command" do
|
12
|
-
context "'all'
|
13
|
-
context "'general'
|
12
|
+
context "'all' request_num is NOT defined" do
|
13
|
+
context "'general' request_num is NOT defined" do
|
14
14
|
let(:config_values) { {} }
|
15
15
|
|
16
|
-
it "should return default
|
17
|
-
|
18
|
-
conf.
|
16
|
+
it "should return default request_num" do
|
17
|
+
default_request_num = conf.send(:defaults)["request_num"]["general"]
|
18
|
+
conf.request_num("all").should eq(default_request_num)
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
22
|
-
context "'general'
|
23
|
-
let(:
|
24
|
-
let(:config_values) { { "
|
22
|
+
context "'general' request_num is defined" do
|
23
|
+
let(:general_request_num) { 20 }
|
24
|
+
let(:config_values) { { "request_num" => { "general" => general_request_num } } }
|
25
25
|
|
26
|
-
it "should return defined 'general'
|
27
|
-
conf.
|
26
|
+
it "should return defined 'general' request_num" do
|
27
|
+
conf.request_num("all").should eq(general_request_num)
|
28
28
|
end
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
32
|
-
context "'all'
|
33
|
-
context "'general'
|
34
|
-
let(:
|
35
|
-
let(:config_values) { { "
|
32
|
+
context "'all' request_num is defined" do
|
33
|
+
context "'general' request_num is NOT defined" do
|
34
|
+
let(:all_request_num) { 20 }
|
35
|
+
let(:config_values) { { "request_num" => { "all" => all_request_num } } }
|
36
36
|
|
37
|
-
it "should return defined 'all'
|
38
|
-
conf.
|
37
|
+
it "should return defined 'all' request_num" do
|
38
|
+
conf.request_num("all").should eq(all_request_num)
|
39
39
|
end
|
40
40
|
end
|
41
41
|
|
42
|
-
context "'general'
|
43
|
-
let(:
|
44
|
-
let(:
|
45
|
-
let(:config_values) { { "
|
42
|
+
context "'general' request_num is defined" do
|
43
|
+
let(:all_request_num) { 20 }
|
44
|
+
let(:general_request_num) { 15 }
|
45
|
+
let(:config_values) { { "request_num" => { "all" => all_request_num, "general" => general_request_num } } }
|
46
46
|
|
47
|
-
it "should return defined 'all'
|
48
|
-
conf.
|
47
|
+
it "should return defined 'all' request_num" do
|
48
|
+
conf.request_num("all").should eq(all_request_num)
|
49
49
|
end
|
50
50
|
end
|
51
51
|
end
|
52
52
|
end
|
53
53
|
end
|
54
54
|
|
55
|
-
describe "#
|
55
|
+
describe "#limit_page" do
|
56
56
|
context "with 'all' command" do
|
57
|
-
context "'all'
|
58
|
-
context "'general'
|
59
|
-
let(:
|
60
|
-
let(:
|
61
|
-
let(:config_values) { { "
|
57
|
+
context "'all' limit_page is defined" do
|
58
|
+
context "'general' limit_page is defined" do
|
59
|
+
let(:all_limit_page) { 10 }
|
60
|
+
let(:general_limit_page) { 5 }
|
61
|
+
let(:config_values) { { "limit_page" => { "all" => all_limit_page, "general" => general_limit_page } } }
|
62
62
|
|
63
|
-
it "should return defined 'all'
|
64
|
-
conf.
|
63
|
+
it "should return defined 'all' limit_page" do
|
64
|
+
conf.limit_page("all").should eq(all_limit_page)
|
65
65
|
end
|
66
66
|
end
|
67
67
|
end
|
data/spec/nicorepo_spec.rb
CHANGED
@@ -1,14 +1,10 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe Nicorepo do
|
4
|
-
|
5
|
-
include Helper
|
6
|
-
|
7
4
|
before(:all) do
|
5
|
+
mail, pass = Netrc.read["nicovideo.jp"]
|
8
6
|
@nicorepo = Nicorepo.new
|
9
|
-
@
|
10
|
-
|
11
|
-
@nicorepo.login(@account[:mail], @account[:pass])
|
7
|
+
@nicorepo.login(mail, pass)
|
12
8
|
end
|
13
9
|
|
14
10
|
describe "#login" do
|
@@ -17,103 +13,49 @@ describe Nicorepo do
|
|
17
13
|
@nicorepo.agent.should be_true
|
18
14
|
end
|
19
15
|
end
|
20
|
-
|
16
|
+
|
21
17
|
context "with wrong account" do
|
22
18
|
it "should raise error" do
|
23
19
|
repo = Nicorepo.new
|
24
|
-
|
20
|
+
expect{ repo.login('test', 'testpass') }.to raise_error(Nicorepo::LoginError)
|
25
21
|
end
|
26
22
|
end
|
27
|
-
|
23
|
+
end
|
28
24
|
|
29
25
|
describe "#all" do
|
30
|
-
context "without arguments" do
|
31
|
-
it "should return 20 logs" do
|
32
|
-
@nicorepo.all.should have(20).logs
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
26
|
context "with 5" do
|
37
|
-
it "should return 5
|
38
|
-
@nicorepo.all(5).should have(5).
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
context "with 30" do
|
43
|
-
it "should return 30 logs" do
|
44
|
-
@nicorepo.all(30).should have(30).logs
|
27
|
+
it "should return 5 reports" do
|
28
|
+
@nicorepo.all(5).should have(5).reports
|
45
29
|
end
|
46
30
|
end
|
47
31
|
|
48
32
|
context "with 50" do
|
49
|
-
it "should return 50
|
50
|
-
@nicorepo.all(50).should have(50).
|
33
|
+
it "should return 50 reports" do
|
34
|
+
@nicorepo.all(50).should have(50).reports
|
51
35
|
end
|
52
36
|
end
|
53
37
|
end
|
54
38
|
|
55
39
|
describe "#videos" do
|
56
|
-
context "
|
57
|
-
it "should return
|
58
|
-
@nicorepo.videos.should have_at_most(3).logs
|
59
|
-
end
|
60
|
-
|
61
|
-
it "should return only video logs" do
|
40
|
+
context "with request_num = 5, limit_page = 3" do
|
41
|
+
it "should return only video reports" do
|
62
42
|
videos = @nicorepo.videos
|
63
|
-
|
64
|
-
|
43
|
+
not_videos = videos.reject{ |v| v.kind =~ /video/ }
|
44
|
+
not_videos.size.should eq 0
|
65
45
|
end
|
66
|
-
end
|
67
46
|
|
68
|
-
|
69
|
-
|
70
|
-
@nicorepo.videos(5, 3).should have_at_most(5).logs
|
47
|
+
it "should return 5 reports at the most" do
|
48
|
+
@nicorepo.videos(5, 3).should have_at_most(5).reports
|
71
49
|
end
|
72
50
|
end
|
73
51
|
end
|
74
52
|
|
75
53
|
describe "#lives" do
|
76
|
-
it "should return only live
|
54
|
+
it "should return only live reports" do
|
77
55
|
lives = @nicorepo.lives
|
78
|
-
|
79
|
-
|
56
|
+
not_lives = lives.reject{ |l| l.kind =~ /live/ }
|
57
|
+
not_lives.size.should eq 0
|
80
58
|
end
|
81
59
|
end
|
82
|
-
|
83
|
-
describe Nicorepo::Log do
|
84
|
-
before(:all) do
|
85
|
-
@log = @nicorepo.all.first
|
86
|
-
p @log
|
87
|
-
end
|
88
|
-
|
89
|
-
it "should have the body" do
|
90
|
-
@log.body.should be_true
|
91
|
-
end
|
92
|
-
|
93
|
-
it "should have the title" do
|
94
|
-
@log.title.should be_true
|
95
|
-
end
|
96
|
-
|
97
|
-
it "should have the url" do
|
98
|
-
@log.url.should be_true
|
99
|
-
end
|
100
|
-
|
101
|
-
it "should have the author" do
|
102
|
-
@log.author.should be_true
|
103
|
-
end
|
104
|
-
|
105
|
-
it "should have the log-kind" do
|
106
|
-
@log.kind.should be_true
|
107
|
-
end
|
108
|
-
|
109
|
-
it "should have the date" do
|
110
|
-
@log.date.should be_true
|
111
|
-
end
|
112
|
-
|
113
|
-
after(:all) do
|
114
|
-
end
|
115
|
-
end
|
116
|
-
|
117
|
-
after(:all) do
|
118
|
-
end
|
119
60
|
end
|
61
|
+
|
data/spec/spec_helper.rb
CHANGED
@@ -1,17 +1,4 @@
|
|
1
1
|
require 'nicorepo'
|
2
2
|
require 'pry'
|
3
|
-
|
4
|
-
module Helper
|
5
|
-
def right_account
|
6
|
-
begin
|
7
|
-
f = open('spec/resource/account.txt')
|
8
|
-
mail = f.gets.chomp!
|
9
|
-
pass = f.gets.chomp!
|
10
|
-
ensure
|
11
|
-
f.close
|
12
|
-
end
|
13
|
-
|
14
|
-
{mail: mail, pass: pass}
|
15
|
-
end
|
16
|
-
end
|
3
|
+
require 'netrc'
|
17
4
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nicorepo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- upinetree
|
@@ -127,11 +127,12 @@ files:
|
|
127
127
|
- lib/nicorepo.rb
|
128
128
|
- lib/nicorepo/cli/cli.rb
|
129
129
|
- lib/nicorepo/cli/config.rb
|
130
|
-
- lib/nicorepo/
|
130
|
+
- lib/nicorepo/parser.rb
|
131
|
+
- lib/nicorepo/report.rb
|
132
|
+
- lib/nicorepo/reports.rb
|
131
133
|
- lib/nicorepo/version.rb
|
132
134
|
- nicorepo.gemspec
|
133
135
|
- spec/cli/cli_config_spec.rb
|
134
|
-
- spec/cli/cli_spec.rb
|
135
136
|
- spec/nicorepo_spec.rb
|
136
137
|
- spec/spec_helper.rb
|
137
138
|
homepage: https://github.com/upinetree/nicorepo
|
@@ -160,6 +161,5 @@ specification_version: 4
|
|
160
161
|
summary: Simple nicorepo scraper
|
161
162
|
test_files:
|
162
163
|
- spec/cli/cli_config_spec.rb
|
163
|
-
- spec/cli/cli_spec.rb
|
164
164
|
- spec/nicorepo_spec.rb
|
165
165
|
- spec/spec_helper.rb
|
data/spec/cli/cli_spec.rb
DELETED
@@ -1,85 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Nicorepo::Cli do
|
4
|
-
before(:all) do
|
5
|
-
@old_stderr = $stderr
|
6
|
-
$stderr = StringIO.new
|
7
|
-
end
|
8
|
-
|
9
|
-
describe "#run" do
|
10
|
-
|
11
|
-
let(:cli) { Nicorepo::Cli.new }
|
12
|
-
before(:each) { cli.stub(:configure) }
|
13
|
-
|
14
|
-
context "when login failed" do
|
15
|
-
it "should exit with error massage" do
|
16
|
-
begin
|
17
|
-
Nicorepo::Cli::Config.any_instance.stub(:account).and_return({mail: "invalid@mail.com", pass: "invalid_pass"})
|
18
|
-
cli.run([ 'i' ])
|
19
|
-
rescue SystemExit => se
|
20
|
-
se.status.should eq 1
|
21
|
-
$stderr.string.should match /invalid mail or pass/
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
context "when login succeed" do
|
27
|
-
|
28
|
-
before(:each) do
|
29
|
-
cli.stub(:login)
|
30
|
-
end
|
31
|
-
|
32
|
-
context "with command 'i'" do
|
33
|
-
it "should exec interactive mode" do
|
34
|
-
cli.should_receive(:interactive_run)
|
35
|
-
cli.run([ 'i' ])
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
context "with command 'lives'" do
|
40
|
-
it "should recieve logs of live" do
|
41
|
-
Nicorepo.any_instance.should_receive(:lives).and_return([Nicorepo::Log.new])
|
42
|
-
cli.run([ 'lives' ])
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
describe "#interactive_run" do
|
49
|
-
context "when entered 'exit'" do
|
50
|
-
it "should return true" do
|
51
|
-
cli = Nicorepo::Cli.new
|
52
|
-
Readline.stub!(:readline) { 'exit' }
|
53
|
-
cli.interactive_run.should be_true
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
describe "#open_url" do
|
59
|
-
context "with 1" do
|
60
|
-
before do
|
61
|
-
@cli = Nicorepo::Cli.new
|
62
|
-
@logs = [ Nicorepo::Log.new ]
|
63
|
-
end
|
64
|
-
|
65
|
-
it "should succeed to open url in browser with first log's url" do
|
66
|
-
@logs.first.url = 'http://www.nicovideo.jp'
|
67
|
-
expect{ @cli.open_url(@logs, 1, {dry_run: true}) }.to be_true
|
68
|
-
end
|
69
|
-
|
70
|
-
it "should raise error when url is nil" do
|
71
|
-
@logs.first.url = nil
|
72
|
-
expect{ @cli.open_url(@logs, 1, {dry_run: true}) }.to raise_error
|
73
|
-
end
|
74
|
-
|
75
|
-
it "should raise error when url is wrongs" do
|
76
|
-
@logs.first.url = 'hoge://piyo'
|
77
|
-
expect{ @cli.open_url(@logs, 1, {dry_run: true}) }.to raise_error
|
78
|
-
end
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
after(:all) do
|
83
|
-
$stderr = @old_stderr
|
84
|
-
end
|
85
|
-
end
|