nicorepo 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +0 -1
- data/README.md +6 -0
- data/bin/nicorepo +2 -2
- data/exe/nicorepo +5 -0
- data/lib/nicorepo/parser.rb +8 -8
- data/lib/nicorepo/version.rb +1 -1
- data/nicorepo.gemspec +6 -3
- metadata +7 -12
- data/spec/cli/cli_config_spec.rb +0 -71
- data/spec/nicorepo_spec.rb +0 -85
- data/spec/spec_helper.rb +0 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5501c9e31cdda24fbcf1bb355856b4f99a293bfd
|
4
|
+
data.tar.gz: e496adc8058c169ef52d5c3a808e22375d386d94
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e8701ba39746179bfd8f4e0d7203acfd03e4ccd2b41c23b5f218c1ac1ac14cead86291d0606ff84af8136e407e2591e2d66f16bb3f042989b751d6c2c4c6d9a4
|
7
|
+
data.tar.gz: ebd6d0210c61adaf63eaa39642040299296dd5898f94d25944736ce310b83fc201668cd192f920edc7c8333ec995e5dae30cf2f326ea94f433b1ba358652a8f3
|
data/Gemfile
CHANGED
data/README.md
CHANGED
data/bin/nicorepo
CHANGED
data/exe/nicorepo
ADDED
data/lib/nicorepo/parser.rb
CHANGED
@@ -33,27 +33,27 @@ class Nicorepo
|
|
33
33
|
end
|
34
34
|
|
35
35
|
def report_nodes(page)
|
36
|
-
page.search('
|
36
|
+
page.search('.timeline/.log')
|
37
37
|
end
|
38
38
|
|
39
39
|
def next_url(page)
|
40
|
-
page.search('
|
40
|
+
page.search('.next-page-link').first['href']
|
41
41
|
end
|
42
42
|
|
43
43
|
def parse_body(node)
|
44
|
-
node.search('
|
44
|
+
node.search('.log-body').first.inner_text.gsub(/(\t|\r|\n)/, "")
|
45
45
|
end
|
46
46
|
|
47
47
|
def parse_title(node)
|
48
|
-
node.search('
|
48
|
+
node.search('.log-target-info/a').first.inner_text
|
49
49
|
end
|
50
50
|
|
51
51
|
def parse_url(node)
|
52
|
-
node.search('
|
52
|
+
node.search('.log-target-info/a').first['href']
|
53
53
|
end
|
54
54
|
|
55
55
|
def parse_author(node)
|
56
|
-
node.search('
|
56
|
+
node.search('.log-body/a').first.inner_text
|
57
57
|
end
|
58
58
|
|
59
59
|
# example: 'log.log-community-video-upload' -> 'community-video-upload'
|
@@ -69,14 +69,14 @@ class Nicorepo
|
|
69
69
|
end
|
70
70
|
|
71
71
|
def parse_date(node)
|
72
|
-
d = node.search('
|
72
|
+
d = node.search('.log-footer-date/time').first['datetime']
|
73
73
|
Time.xmlschema(d).localtime
|
74
74
|
end
|
75
75
|
|
76
76
|
def error_report(node, e)
|
77
77
|
{
|
78
78
|
body: node.inner_html.gsub(/\r|\n|\t/, ''),
|
79
|
-
title: "An exception occured: #{e.message}",
|
79
|
+
title: "An exception occured: #{e.message}\n#{e.backtrace}",
|
80
80
|
url: '',
|
81
81
|
author: '',
|
82
82
|
kind: Report::ERROR_KIND,
|
data/lib/nicorepo/version.rb
CHANGED
data/nicorepo.gemspec
CHANGED
@@ -6,14 +6,17 @@ Gem::Specification.new do |spec|
|
|
6
6
|
spec.name = "nicorepo"
|
7
7
|
spec.version = Nicorepo::VERSION
|
8
8
|
spec.authors = ["upinetree"]
|
9
|
-
spec.email = ["
|
9
|
+
spec.email = ["upinetree@gmail.com"]
|
10
10
|
spec.summary = %q{Simple nicorepo scraper}
|
11
11
|
spec.description = %q{Scrapes and filters nicorepo reports from nicovideo.}
|
12
12
|
spec.homepage = "https://github.com/upinetree/nicorepo"
|
13
13
|
spec.license = "MIT"
|
14
14
|
|
15
|
-
spec.files = `git ls-files -z`.split("\x0")
|
16
|
-
|
15
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
16
|
+
f.match(%r{^(test|spec|features)/})
|
17
|
+
end
|
18
|
+
spec.bindir = "exe"
|
19
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
17
20
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
21
|
spec.require_paths = ["lib"]
|
19
22
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nicorepo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- upinetree
|
8
8
|
autorequire:
|
9
|
-
bindir:
|
9
|
+
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-05-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mechanize
|
@@ -124,7 +124,7 @@ dependencies:
|
|
124
124
|
version: '0'
|
125
125
|
description: Scrapes and filters nicorepo reports from nicovideo.
|
126
126
|
email:
|
127
|
-
-
|
127
|
+
- upinetree@gmail.com
|
128
128
|
executables:
|
129
129
|
- nicorepo
|
130
130
|
extensions: []
|
@@ -139,6 +139,7 @@ files:
|
|
139
139
|
- README.md
|
140
140
|
- Rakefile
|
141
141
|
- bin/nicorepo
|
142
|
+
- exe/nicorepo
|
142
143
|
- lib/nicorepo.rb
|
143
144
|
- lib/nicorepo/cli/cli.rb
|
144
145
|
- lib/nicorepo/cli/config.rb
|
@@ -147,9 +148,6 @@ files:
|
|
147
148
|
- lib/nicorepo/reports.rb
|
148
149
|
- lib/nicorepo/version.rb
|
149
150
|
- nicorepo.gemspec
|
150
|
-
- spec/cli/cli_config_spec.rb
|
151
|
-
- spec/nicorepo_spec.rb
|
152
|
-
- spec/spec_helper.rb
|
153
151
|
homepage: https://github.com/upinetree/nicorepo
|
154
152
|
licenses:
|
155
153
|
- MIT
|
@@ -170,11 +168,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
170
168
|
version: '0'
|
171
169
|
requirements: []
|
172
170
|
rubyforge_project:
|
173
|
-
rubygems_version: 2.
|
171
|
+
rubygems_version: 2.6.11
|
174
172
|
signing_key:
|
175
173
|
specification_version: 4
|
176
174
|
summary: Simple nicorepo scraper
|
177
|
-
test_files:
|
178
|
-
- spec/cli/cli_config_spec.rb
|
179
|
-
- spec/nicorepo_spec.rb
|
180
|
-
- spec/spec_helper.rb
|
175
|
+
test_files: []
|
data/spec/cli/cli_config_spec.rb
DELETED
@@ -1,71 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Nicorepo::Cli::Configuration do
|
4
|
-
let(:conf) { Nicorepo::Cli::Configuration.new }
|
5
|
-
|
6
|
-
before do
|
7
|
-
allow_any_instance_of(Nicorepo::Cli::Configuration).to receive(:load_config).and_return(config_values)
|
8
|
-
end
|
9
|
-
|
10
|
-
describe "#request_num" do
|
11
|
-
context "with 'all' command" do
|
12
|
-
context "'all' request_num is NOT defined" do
|
13
|
-
context "'general' request_num is NOT defined" do
|
14
|
-
let(:config_values) { {} }
|
15
|
-
|
16
|
-
it "should return default request_num" do
|
17
|
-
default_request_num = conf.send(:defaults)["request_num"]["general"]
|
18
|
-
expect(conf.request_num("all")).to eq(default_request_num)
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
context "'general' request_num is defined" do
|
23
|
-
let(:general_request_num) { 20 }
|
24
|
-
let(:config_values) { { "request_num" => { "general" => general_request_num } } }
|
25
|
-
|
26
|
-
it "should return defined 'general' request_num" do
|
27
|
-
expect(conf.request_num("all")).to eq(general_request_num)
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
context "'all' request_num is defined" do
|
33
|
-
context "'general' request_num is NOT defined" do
|
34
|
-
let(:all_request_num) { 20 }
|
35
|
-
let(:config_values) { { "request_num" => { "all" => all_request_num } } }
|
36
|
-
|
37
|
-
it "should return defined 'all' request_num" do
|
38
|
-
expect(conf.request_num("all")).to eq(all_request_num)
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
context "'general' request_num is defined" do
|
43
|
-
let(:all_request_num) { 20 }
|
44
|
-
let(:general_request_num) { 15 }
|
45
|
-
let(:config_values) { { "request_num" => { "all" => all_request_num, "general" => general_request_num } } }
|
46
|
-
|
47
|
-
it "should return defined 'all' request_num" do
|
48
|
-
expect(conf.request_num("all")).to eq(all_request_num)
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
describe "#limit_page" do
|
56
|
-
context "with 'all' command" do
|
57
|
-
context "'all' limit_page is defined" do
|
58
|
-
context "'general' limit_page is defined" do
|
59
|
-
let(:all_limit_page) { 10 }
|
60
|
-
let(:general_limit_page) { 5 }
|
61
|
-
let(:config_values) { { "limit_page" => { "all" => all_limit_page, "general" => general_limit_page } } }
|
62
|
-
|
63
|
-
it "should return defined 'all' limit_page" do
|
64
|
-
expect(conf.limit_page("all")).to eq(all_limit_page)
|
65
|
-
end
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
data/spec/nicorepo_spec.rb
DELETED
@@ -1,85 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Nicorepo do
|
4
|
-
before(:all) do
|
5
|
-
mail, pass = Netrc.read["nicovideo.jp"]
|
6
|
-
@nicorepo = Nicorepo.new
|
7
|
-
@nicorepo.login(mail, pass)
|
8
|
-
end
|
9
|
-
|
10
|
-
shared_examples_for 'an error report in parsing' do
|
11
|
-
context "when an error occured" do
|
12
|
-
before do
|
13
|
-
allow_any_instance_of(Nicorepo::Parser).to receive(:parse_title).and_raise("SomeError")
|
14
|
-
end
|
15
|
-
|
16
|
-
it "should return erorr reports" do
|
17
|
-
expect(report.title).to eq("An exception occured: SomeError")
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
describe "#login" do
|
23
|
-
context "with right account" do
|
24
|
-
it "should be success" do
|
25
|
-
expect(@nicorepo.agent).to be_truthy
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
context "with wrong account" do
|
30
|
-
it "should raise error" do
|
31
|
-
repo = Nicorepo.new
|
32
|
-
expect{ repo.login('test', 'testpass') }.to raise_error(Nicorepo::LoginError)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
describe "#all" do
|
38
|
-
context "with 5" do
|
39
|
-
it "should return 5 reports" do
|
40
|
-
expect(@nicorepo.all(5).size).to eq(5)
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
context "with 50" do
|
45
|
-
it "should return 50 reports" do
|
46
|
-
expect(@nicorepo.all(50).size).to eq(50)
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
it_behaves_like 'an error report in parsing' do
|
51
|
-
let(:report) { @nicorepo.all(1).first }
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
describe "#videos" do
|
56
|
-
context "with request_num = 5, limit_page = 3" do
|
57
|
-
it "should return only video reports" do
|
58
|
-
videos = @nicorepo.videos
|
59
|
-
not_videos = videos.reject{ |v| v.kind =~ /video/ }
|
60
|
-
expect(not_videos.size).to eq(0)
|
61
|
-
end
|
62
|
-
|
63
|
-
it "should return 5 reports at the most" do
|
64
|
-
expect(@nicorepo.videos(5, 3).size).to be <= 5
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
it_behaves_like 'an error report in parsing' do
|
69
|
-
let(:report) { @nicorepo.videos(1).first }
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
describe "#lives" do
|
74
|
-
it "should return only live reports" do
|
75
|
-
lives = @nicorepo.lives
|
76
|
-
not_lives = lives.reject{ |l| l.kind =~ /live/ }
|
77
|
-
expect(not_lives.size).to eq(0)
|
78
|
-
end
|
79
|
-
|
80
|
-
it_behaves_like 'an error report in parsing' do
|
81
|
-
let(:report) { @nicorepo.lives(1).first }
|
82
|
-
end
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
data/spec/spec_helper.rb
DELETED