consadole_aggregator 0.1.9 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -0
- data/.rspec +2 -1
- data/.rvmrc +1 -0
- data/.travis.yml +1 -0
- data/Gemfile +3 -9
- data/Gemfile.lock +52 -27
- data/Guardfile +19 -0
- data/Rakefile +3 -49
- data/bin/consadole_aggregator +29 -49
- data/consadole_aggregator.gemspec +22 -210
- data/lib/consadole_aggregator/helper.rb +8 -4
- data/lib/consadole_aggregator/live.rb +74 -67
- data/lib/consadole_aggregator/news.rb +165 -100
- data/lib/consadole_aggregator/version.rb +3 -0
- data/lib/consadole_aggregator.rb +26 -4
- data/spec/consadole_aggregator/helper_spec.rb +13 -8
- data/spec/consadole_aggregator/live_spec.rb +68 -124
- data/spec/consadole_aggregator/news_spec.rb +70 -148
- data/spec/consadole_aggregator_spec.rb +5 -0
- data/spec/ext/clubconsadole.txt +262 -315
- data/spec/ext/jsgoalphotos.txt +321 -321
- data/spec/spec_helper.rb +13 -9
- metadata +233 -58
- data/.document +0 -5
- data/LICENSE.txt +0 -20
- data/README.rdoc +0 -33
- data/VERSION +0 -1
- data/account.yaml +0 -6
- data/db/.gitignore +0 -0
- data/lib/consadole_aggregator/aggregatable.rb +0 -62
- data/lib/consadole_aggregator/live/timeline.rb +0 -13
- data/log/.gitignore +0 -0
- data/spec/consadole_aggregator/aggregatable_spec.rb +0 -87
- data/spec/consadole_aggregator/live/timeline_spec.rb +0 -53
- data/spec/spec.opts +0 -1
@@ -1,145 +1,89 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
|
-
|
2
|
+
require 'spec_helper'
|
3
|
+
require 'time'
|
3
4
|
|
4
|
-
|
5
|
-
|
6
|
-
describe ConsadoleAggregator do
|
5
|
+
module ConsadoleAggregator
|
7
6
|
describe Live do
|
8
|
-
|
9
|
-
|
10
|
-
Net::HTTP.should_receive(:get)
|
11
|
-
.with(URI.parse('http://www.consadole-sapporo.jp/view/s674.html'))
|
12
|
-
.and_return(File.read(File.dirname(__FILE__) + '/../ext/live/s674.html'))
|
13
|
-
Live::Live.get_resource.call
|
14
|
-
end
|
7
|
+
before do
|
8
|
+
ConsadoleAggregator.stub(:logger).and_return(double('logger').as_null_object)
|
15
9
|
end
|
10
|
+
let(:document) { File.read(ext_path('live/s674.html.120')).force_encoding('SJIS') }
|
16
11
|
|
17
|
-
describe
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
subject{ parsed_list }
|
22
|
-
it{ should have(3).items }
|
23
|
-
end
|
24
|
-
context 'when end of game' do
|
25
|
-
let(:resource){ File.read(File.dirname(__FILE__) + '/../ext/live/s674.html.120').toutf8 }
|
26
|
-
let(:parsed_list){ Live::Live.parse_list.call(resource) }
|
27
|
-
describe 'first TimeLine' do
|
28
|
-
subject{ parsed_list.first }
|
29
|
-
it{ should == '試合開始 札幌ボールでキックオフ' }
|
30
|
-
end
|
31
|
-
describe 'second TimeLine' do
|
32
|
-
subject{ parsed_list[1] }
|
33
|
-
it{ should == '1分 右サイドからボールをつながれ攻撃を仕掛けられるが札幌DFが落ち着いてクリア' }
|
34
|
-
end
|
35
|
-
describe 'last TimeLine' do
|
36
|
-
subject{ parsed_list.last }
|
37
|
-
it{ should == '試合終了 ロスタイムも余裕のプレーで相手の攻撃を許さず、3試合連続完封で3連勝を飾る' }
|
38
|
-
end
|
39
|
-
end
|
12
|
+
describe ".parse" do
|
13
|
+
it { described_class.parse(document).should have(69).items }
|
14
|
+
it { described_class.parse(document).first.should eq '試合開始 札幌ボールでキックオフ' }
|
15
|
+
it { described_class.parse(document).last.should be_include '試合終了' }
|
40
16
|
end
|
41
17
|
|
42
|
-
describe
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
describe '.reserve' do
|
48
|
-
context 'given Time' do
|
49
|
-
it 'give constructor with Time ' do
|
50
|
-
Live::Live.should_receive(:new).with(Time.parse('2011-02-14 13:00'), {})
|
51
|
-
Live.reserve(Time.parse('2011-02-14 13:00'))
|
52
|
-
end
|
18
|
+
describe ".fetch" do
|
19
|
+
context "when HTTP.get success" do
|
20
|
+
before { Net::HTTP.stub(:get).and_return(document) }
|
21
|
+
it { described_class.fetch.should have_at_least(1).items }
|
53
22
|
end
|
54
|
-
end
|
55
|
-
end
|
56
23
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
context 'when normal update' do
|
61
|
-
subject{ Live::Live.new }
|
62
|
-
it 'should to be be_daemonize' do
|
63
|
-
subject.should_receive(:be_daemonize).ordered
|
64
|
-
subject.should_receive(:wait_initial).ordered
|
65
|
-
subject.should_receive(:update).ordered.exactly(240).times
|
66
|
-
subject.should_receive(:sleep).with(30).exactly(240).times
|
67
|
-
subject.execute
|
68
|
-
end
|
69
|
-
end
|
70
|
-
context 'when raise Exception' do
|
71
|
-
before do
|
72
|
-
@live = Live::Live.new(nil, { times:1 })
|
73
|
-
@live.stub!(:be_daemonize)
|
74
|
-
@live.stub!(:wait_initial)
|
75
|
-
end
|
76
|
-
subject{ @live }
|
77
|
-
it 'should log exception and sleep' do
|
78
|
-
subject.should_receive(:sleep).once
|
79
|
-
subject.execute{ |timeline| raise }
|
80
|
-
end
|
24
|
+
context "when HTTP.get failure" do
|
25
|
+
before { Net::HTTP.stub(:get).and_raise("can't get") }
|
26
|
+
it { described_class.fetch.should have(0).items }
|
81
27
|
end
|
82
28
|
end
|
83
29
|
|
84
|
-
describe
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
subject
|
95
|
-
it 'sleep 36000 sec' do
|
96
|
-
subject.should_receive(:sleep).with(1.0*60*60*10)
|
97
|
-
subject.wait_initial
|
98
|
-
end
|
30
|
+
describe "#update" do
|
31
|
+
let(:fetched) {
|
32
|
+
[
|
33
|
+
'試合開始 札幌ボールでキックオフ',
|
34
|
+
'1分 【札幌先制GOAL!】',
|
35
|
+
'2分 【札幌GOAL!】'
|
36
|
+
]
|
37
|
+
}
|
38
|
+
before do
|
39
|
+
described_class.stub(:fetch).and_return(fetched)
|
40
|
+
subject.send(:posted) << fetched.first
|
99
41
|
end
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
42
|
+
|
43
|
+
it 'when first called raise error,other times don\'t raise error' do
|
44
|
+
result = []
|
45
|
+
called = false
|
46
|
+
subject.update { |post|
|
47
|
+
if called
|
48
|
+
result << post
|
49
|
+
else
|
50
|
+
called = true
|
51
|
+
raise
|
52
|
+
end
|
53
|
+
}
|
54
|
+
result.should eq [{ title: fetched.last}]
|
107
55
|
end
|
108
56
|
end
|
109
57
|
|
110
|
-
describe
|
58
|
+
describe ".run" do
|
59
|
+
let(:runner) { double('runner').as_null_object }
|
60
|
+
let(:starting_time) { Time.parse('2012-04-07 20:48') }
|
61
|
+
let(:option) {
|
62
|
+
{
|
63
|
+
length_of_a_game: 180,
|
64
|
+
interval: 30,
|
65
|
+
daemonize: false
|
66
|
+
}
|
67
|
+
}
|
68
|
+
subject { described_class.run starting_time, option }
|
111
69
|
before do
|
112
|
-
|
113
|
-
[
|
114
|
-
Live::Timeline.parse('1分 右サイドからボールをつながれ攻撃を仕掛けられるが札幌DFが落ち着いてクリア'),
|
115
|
-
Live::Timeline.parse('2分 左サイドキリノのパスカットから攻撃を仕掛けるがシュートまでは持ち込めず'),
|
116
|
-
Live::Timeline.parse('3分 ゴール前ほぼ正面やや遠めのFKを上里が直接狙うが湘南DFの壁に当たる'),
|
117
|
-
]
|
118
|
-
end
|
119
|
-
context 'when first time' do
|
120
|
-
before { Live.stub!(:parse).and_return(@first_timeline) }
|
121
|
-
subject{ Live::Live.new }
|
122
|
-
it{ expect{ subject.update }.to change{ subject.posted.dup }.from([]).to(@first_timeline) }
|
123
|
-
end
|
124
|
-
context 'when second time' do
|
125
|
-
before do
|
126
|
-
@second_timeline = @first_timeline.clone.push(Live::Timeline.parse('3分 右サイドからのクロスに阿部(湘南)がヘッドであわせるがGK高原がキャッチ'))
|
127
|
-
Live.stub!(:parse).and_return(@first_timeline, @second_timeline)
|
128
|
-
@live = Live::Live.new
|
129
|
-
@live.update
|
130
|
-
end
|
131
|
-
subject{ @live }
|
132
|
-
it { expect { subject.update }.to change{ subject.posted.dup }.from(@first_timeline).to(@second_timeline) }
|
133
|
-
end
|
134
|
-
context 'given block' do
|
135
|
-
subject{ Live::Live.new }
|
136
|
-
before do
|
137
|
-
Live.stub!(:parse).and_return(@first_timeline)
|
138
|
-
@ary = []
|
139
|
-
subject.update { |timeline| @ary << timeline }
|
140
|
-
end
|
141
|
-
it { @ary.should == @first_timeline }
|
70
|
+
Live::Runner.stub(:new).and_return(runner)
|
142
71
|
end
|
72
|
+
|
73
|
+
it {
|
74
|
+
runner.should_receive(:length_of_a_game=).with(180)
|
75
|
+
subject
|
76
|
+
}
|
77
|
+
|
78
|
+
it {
|
79
|
+
runner.should_receive(:interval=).with(30)
|
80
|
+
subject
|
81
|
+
}
|
82
|
+
|
83
|
+
it {
|
84
|
+
runner.should_receive(:daemonize=).with(false)
|
85
|
+
subject
|
86
|
+
}
|
143
87
|
end
|
144
88
|
end
|
145
89
|
end
|
@@ -1,167 +1,89 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
|
-
|
2
|
+
require 'spec_helper'
|
3
|
+
require 'stringio'
|
3
4
|
|
4
|
-
|
5
|
-
|
6
|
-
describe ConsadoleAggregator do
|
7
|
-
describe Aggregatable do
|
5
|
+
module ConsadoleAggregator
|
6
|
+
describe News do
|
8
7
|
before do
|
9
|
-
|
10
|
-
{ url:'http://example.com/', title:'fuga' }]
|
11
|
-
get_resource_stub = double('get_resource')
|
12
|
-
get_resource_stub.stub(:call).and_return('')
|
13
|
-
parse_list_stub = double('parse_list')
|
14
|
-
parse_list_stub.stub(:call).and_return(['http://example.jp/', 'http://example.com/'])
|
15
|
-
parse_article_stub = double('parse_article')
|
16
|
-
parse_article_stub.stub(:call) do |arg|
|
17
|
-
if arg == 'http://example.jp/'
|
18
|
-
{ url:arg, title:'hoge' }
|
19
|
-
else
|
20
|
-
{ url:arg, title:'fuga' }
|
21
|
-
end
|
22
|
-
end
|
23
|
-
klass = Class.new do
|
24
|
-
include Aggregatable
|
25
|
-
@get_resource = get_resource_stub
|
26
|
-
@parse_list = parse_list_stub
|
27
|
-
@parse_article = parse_article_stub
|
28
|
-
end
|
29
|
-
ConsadoleAggregator::News.const_set(:TestClass, klass) # FIXME How do I suppress warning?
|
30
|
-
subject.stub(:save_strage)
|
8
|
+
ConsadoleAggregator.stub(:logger).and_return(double('logger').as_null_object)
|
31
9
|
end
|
32
10
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
end
|
41
|
-
it 'should return part of articles' do
|
42
|
-
subject.get_new_articles.should == @articles - @straged
|
43
|
-
end
|
44
|
-
end
|
11
|
+
describe '.register' do
|
12
|
+
it { expect {
|
13
|
+
described_class.send(:register!) do |sites|
|
14
|
+
sites.name(:a_site)
|
15
|
+
end
|
16
|
+
}.to change { News::Sites.size }.by(1)
|
17
|
+
}
|
45
18
|
end
|
19
|
+
end
|
46
20
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
subject.update
|
53
|
-
end
|
21
|
+
module News
|
22
|
+
describe Site do
|
23
|
+
before do
|
24
|
+
ConsadoleAggregator.stub(:logger).and_return(double('logger').as_null_object)
|
25
|
+
described_class.stub_chain(:store, :transaction).and_return(updated)
|
54
26
|
end
|
55
|
-
|
56
|
-
|
57
|
-
YAML.stub!(:load_file).and_return([])
|
58
|
-
subject.stub!(:get_new_articles).and_return(@articles)
|
59
|
-
end
|
60
|
-
it 'should add strage' do
|
61
|
-
expect{ subject.update }.to change{ subject.get_strage.dup }.from([]).to(@articles)
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
27
|
+
let(:updated) { [] }
|
28
|
+
subject { described_class.new(:hochiyomiuri) }
|
65
29
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
30
|
+
describe '#update' do
|
31
|
+
let(:site) {
|
32
|
+
site = described_class.new(:hochiyomiuri)
|
33
|
+
site.resource = -> { File.read(ext_path('hochiyomiuri.txt')).encode('UTF-8') }
|
34
|
+
site.list_parser = ->(list) { Nokogiri::HTML(list).search('div.list1 > ul > li a').reverse }
|
35
|
+
site.article_filter = ->(article) { article.text =~ /札幌|コンサ/ }
|
36
|
+
site.article_parser = ->(article) { { url: "http://hochi.yomiuri.co.jp#{article['href']}", title: article.text } }
|
37
|
+
site
|
38
|
+
}
|
39
|
+
let(:block) { ->(article) { result << article } }
|
40
|
+
let(:result){ [] }
|
41
|
+
|
42
|
+
subject { result }
|
43
|
+
|
44
|
+
describe 'element' do
|
45
|
+
before { site.update &block }
|
46
|
+
it { subject.first.should include :url, :title }
|
47
|
+
it { subject.should satisfy { |list| list.all? { |e| e[:title].include?('札幌') || e[:title].include?('コンサ') } } }
|
76
48
|
end
|
77
|
-
it { expect{ subject.get_strage }.to raise_error }
|
78
|
-
end
|
79
|
-
end
|
80
49
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
end
|
86
|
-
end
|
50
|
+
describe 'updated' do
|
51
|
+
before { site.update &block }
|
52
|
+
it { updated.should_not be_empty }
|
53
|
+
end
|
87
54
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
ConsadoleAggregator::News::Consaburn.get_resource =
|
99
|
-
->{ File.read('./spec/ext/consaburn.txt').toutf8 }
|
100
|
-
ConsadoleAggregator::News::Consaclub.get_resource =
|
101
|
-
->{ File.read('./spec/ext/consaclub.txt').toutf8 }
|
102
|
-
ConsadoleAggregator::News::Consadolenews.get_resource =
|
103
|
-
->{ File.read('./spec/ext/consadolenews.txt').toutf8 }
|
104
|
-
ConsadoleAggregator::News::Consadolephotos.get_resource =
|
105
|
-
->{ File.read('./spec/ext/consadolephotos.txt').toutf8 }
|
106
|
-
ConsadoleAggregator::News::Jsgoalnews.get_resource =
|
107
|
-
->{ File.read('./spec/ext/jsgoalnews.txt').toutf8 }
|
108
|
-
ConsadoleAggregator::News::Jsgoalphotos.get_resource =
|
109
|
-
->{ File.read('./spec/ext/jsgoalphotos.txt').toutf8 }
|
110
|
-
ConsadoleAggregator::News::Clubconsadole.get_resource =
|
111
|
-
->{ File.read('./spec/ext/clubconsadole.txt').toutf8 }
|
55
|
+
context 'when have updated item' do
|
56
|
+
before { site.update &block }
|
57
|
+
let(:updated) {
|
58
|
+
resource = File.read(ext_path('hochiyomiuri.txt')).encode('UTF-8')
|
59
|
+
Nokogiri::HTML(resource).search('div.list1 > ul > li a').reverse.slice(0...-1).map { |i|
|
60
|
+
{ url: "http://hochi.yomiuri.co.jp#{i['href']}", title: i.text }
|
61
|
+
}
|
62
|
+
}
|
63
|
+
it { subject.should have(1).item }
|
64
|
+
end
|
112
65
|
|
113
|
-
|
114
|
-
|
115
|
-
|
66
|
+
context 'when raise error on get resource' do
|
67
|
+
before { Net::HTTP.stub(:get).and_return { raise } }
|
68
|
+
it { expect { site.update &block }.not_to raise_error }
|
116
69
|
end
|
117
|
-
end
|
118
|
-
end
|
119
70
|
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
it 'Consaclub should not raise Exception' do
|
136
|
-
expect{ ConsadoleAggregator::News::Consaclub.new.get_new_articles }.to_not raise_error
|
137
|
-
end
|
138
|
-
describe ConsadoleAggregator::News::Consadolenews do
|
139
|
-
subject{ ConsadoleAggregator::News::Consadolenews.new }
|
140
|
-
it 'Consadolenews should not raise Exception' do
|
141
|
-
expect{ subject.get_new_articles }.to_not raise_error
|
142
|
-
end
|
143
|
-
it{ subject.get_new_articles.should have_at_least(1).items }
|
144
|
-
end
|
145
|
-
describe ConsadoleAggregator::News::Consadolephotos do
|
146
|
-
subject{ ConsadoleAggregator::News::Consadolephotos.new }
|
147
|
-
it 'Consadolephotos should not raise Exception' do
|
148
|
-
expect{ subject.get_new_articles }.to_not raise_error
|
149
|
-
end
|
150
|
-
it{ subject.get_new_articles.should have_at_least(1).items }
|
151
|
-
end
|
152
|
-
it 'Jsgoalnews should not raise Exception' do
|
153
|
-
expect{ ConsadoleAggregator::News::Jsgoalnews.new.get_new_articles }.to_not raise_error
|
154
|
-
end
|
155
|
-
it 'Jsgoalphotos should not raise Exception' do
|
156
|
-
expect{ ConsadoleAggregator::News::Jsgoalphotos.new.get_new_articles }.to_not raise_error
|
157
|
-
end
|
158
|
-
describe ConsadoleAggregator::News::Clubconsadole do
|
159
|
-
subject{ ConsadoleAggregator::News::Clubconsadole.new }
|
160
|
-
it 'Clubconsadole should not raise Exception' do
|
161
|
-
expect{ subject.get_new_articles }.to_not raise_error
|
71
|
+
context 'when raise error on block' do
|
72
|
+
before { site.update &block }
|
73
|
+
let(:block) {
|
74
|
+
error_occured = false
|
75
|
+
->(article) {
|
76
|
+
unless error_occured
|
77
|
+
error_occured = true
|
78
|
+
raise
|
79
|
+
end
|
80
|
+
result << article
|
81
|
+
}
|
82
|
+
}
|
83
|
+
it { expect { site.update &block }.not_to raise_error }
|
84
|
+
it { subject.should_not be_empty }
|
85
|
+
end
|
162
86
|
end
|
163
|
-
it{ subject.get_new_articles.should have_at_least(1).items }
|
164
|
-
it{ p subject.get_new_articles }
|
165
87
|
end
|
166
88
|
end
|
167
89
|
end
|