consadole_aggregator 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.4
|
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{consadole_aggregator}
|
8
|
-
s.version = "0.0.
|
8
|
+
s.version = "0.0.4"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["niku"]
|
12
|
-
s.date = %q{
|
12
|
+
s.date = %q{2010-02-15}
|
13
13
|
s.email = %q{niku@niku.name}
|
14
14
|
s.files = [
|
15
15
|
".gitignore",
|
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
|
|
25
25
|
"spec/nikkan_sports/consadole.rdf",
|
26
26
|
"spec/nikkan_sports/nikkan_sports_spec.rb",
|
27
27
|
"spec/nikkan_sports/p-sc-tp0-20091225-579346.html",
|
28
|
+
"spec/nikkan_sports/p-sc-tp0-20100204-592466.html",
|
28
29
|
"spec/spec.opts",
|
29
30
|
"spec/spec_helper.rb",
|
30
31
|
"spec/timeline/s674.html",
|
@@ -1,51 +1,61 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
|
-
require 'kconv'
|
3
2
|
require 'rss'
|
4
3
|
|
5
4
|
require 'rubygems'
|
6
5
|
require 'hpricot' # 2009/12/26 現在 GAE では nokogiri が使えないので hpricot
|
6
|
+
require 'dm-core'
|
7
|
+
require 'dm-types'
|
8
|
+
require 'dm-validations'
|
7
9
|
|
8
|
-
module
|
10
|
+
module Consadole
|
9
11
|
module NikkanSports
|
10
|
-
FEED_URI = URI.parse('http://www.nikkansports.com/rss/soccer/jleague/consadole.rdf')
|
11
|
-
|
12
|
-
def NikkanSports.uris
|
13
|
-
rss = RSS::Parser.parse(FEED_URI, false)
|
14
|
-
rss.items.map(&:link)
|
15
|
-
end
|
16
|
-
|
17
12
|
class Entry
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
end
|
23
|
-
|
24
|
-
def uri
|
25
|
-
URI.parse(@doc.at('a')['href'])
|
26
|
-
end
|
13
|
+
module Parser
|
14
|
+
def self.included mod
|
15
|
+
mod.extend ClassMethods
|
16
|
+
end
|
27
17
|
|
28
|
-
|
29
|
-
|
18
|
+
module ClassMethods
|
19
|
+
def parse uri, doc
|
20
|
+
target = Hpricot(doc)
|
21
|
+
entry = self.new
|
22
|
+
entry.uri = uri
|
23
|
+
entry.title = target.at('h1').inner_text
|
24
|
+
entry.text = target.search('div#news p')[0...-1].to_html.gsub(/【.+】<\/p>$/,'</p>')
|
25
|
+
entry.auther = target.search('div#news p')[0...-1].to_html.match(/【(.+)】<\/p>$/).to_a[1]
|
26
|
+
entry.post_date = Time.strptime(target.search('p.timeStamp').inner_text.match(/\d+年\d+月\d+日\d+時\d+分/).to_s, '%Y年%m月%d日%H時%M分')
|
27
|
+
if tmp = target.at('dl.photo img')
|
28
|
+
source = IMAGE_BASE_URI + tmp['src'].sub(/\.jpg$/, '-big.jpg')
|
29
|
+
entry.image_src = source
|
30
|
+
entry.image_binary = Net::HTTP.get(source)
|
31
|
+
entry.image_alt = tmp['alt']
|
32
|
+
end
|
33
|
+
entry
|
34
|
+
end
|
35
|
+
end
|
30
36
|
end
|
31
37
|
|
32
|
-
|
33
|
-
@doc.search('div#news p')[0...-1].to_html.gsub(/【.+】<\/p>$/,'</p>')
|
34
|
-
end
|
38
|
+
IMAGE_BASE_URI = URI.parse('http://cache2.nipc.jp/soccer/news/')
|
35
39
|
|
36
|
-
|
37
|
-
|
38
|
-
|
40
|
+
include Parser
|
41
|
+
include DataMapper::Resource
|
42
|
+
|
43
|
+
property :id, Serial
|
44
|
+
property :uri, URI, :unique => true
|
45
|
+
property :title, String
|
46
|
+
property :text, Text
|
47
|
+
property :auther, String
|
48
|
+
property :post_date, Time
|
49
|
+
property :image_src, URI
|
50
|
+
property :image_alt, String
|
51
|
+
property :image_binary, Object
|
52
|
+
end
|
39
53
|
|
40
|
-
|
41
|
-
Time.strptime(@doc.search('p.timeStamp').inner_text.match(/\d+年\d+月\d+日\d+時\d+分/).to_s, '%Y年%m月%d日%H時%M分')
|
42
|
-
end
|
54
|
+
FEED_URI = URI.parse('http://www.nikkansports.com/rss/soccer/jleague/consadole.rdf')
|
43
55
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
end
|
48
|
-
end
|
56
|
+
def NikkanSports.uris
|
57
|
+
rss = RSS::Parser.parse(FEED_URI, false)
|
58
|
+
rss.items.map(&:link)
|
49
59
|
end
|
50
60
|
end
|
51
61
|
end
|
data/lib/consadole_aggregator.rb
CHANGED
@@ -19,23 +19,3 @@ if RUBY_VERSION < '1.9'
|
|
19
19
|
end
|
20
20
|
end
|
21
21
|
end
|
22
|
-
|
23
|
-
module ConsadoleAggregator
|
24
|
-
Image = Struct.new :src, :alt, :binary do
|
25
|
-
def binary
|
26
|
-
self['binary'] ||= Net::HTTP.get(self['src']).encode(Encoding::BINARY)
|
27
|
-
end
|
28
|
-
|
29
|
-
def to_html
|
30
|
-
%Q{<image src="#{self.src}" alt="#{self.alt}">}
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
if __FILE__ == $0
|
36
|
-
DataMapper.setup(:default, 'sqlite3:db/test.sqlite3')
|
37
|
-
DataMapper.auto_migrate!
|
38
|
-
|
39
|
-
ConsadoleAggregator::NikkanSports.save
|
40
|
-
print ConsadoleAggregator::NikkanSports.rss
|
41
|
-
end
|
@@ -2,12 +2,14 @@
|
|
2
2
|
require 'time'
|
3
3
|
|
4
4
|
require File.join(File.dirname(__FILE__), '..', 'spec_helper')
|
5
|
-
|
5
|
+
TEST_WITH_IMGAE_FILE_NAME = File.join(File.dirname(__FILE__), 'p-sc-tp0-20091225-579346.html')
|
6
|
+
TEST_WITHOUT_IMAGE_FILE_NAME = File.join(File.dirname(__FILE__), 'p-sc-tp0-20100204-592466.html')
|
7
|
+
TEST_WITHOUT_AUTHER_FILE_NAME = File.join(File.dirname(__FILE__), 'p-sc-tp0-20100204-592466.html')
|
6
8
|
TEST_IMAGE_FILE_NAME = File.join(File.dirname(__FILE__), 'sc-091225-1-ns-big.jpg')
|
7
9
|
|
8
|
-
describe
|
10
|
+
describe Consadole::NikkanSports::Entry do
|
9
11
|
before do
|
10
|
-
@doc =
|
12
|
+
@doc = Consadole::NikkanSports::Entry::parse(nil, File.open(TEST_WITH_IMGAE_FILE_NAME).read)
|
11
13
|
end
|
12
14
|
|
13
15
|
it 'title を実行した場合 ' do
|
@@ -26,26 +28,31 @@ describe ConsadoleAggregator::NikkanSports::Entry do
|
|
26
28
|
@doc.post_date.should == Time.mktime(2009,12,25,8,21)
|
27
29
|
end
|
28
30
|
|
29
|
-
it 'image を実行した場合' do
|
30
|
-
@doc.image.should be_true
|
31
|
-
end
|
32
|
-
|
33
31
|
it 'image.src を実行した場合' do
|
34
|
-
@doc.image.src.should
|
32
|
+
@doc.image.src.should === URI.parse('http://cache2.nipc.jp/soccer/news/img/sc-091225-1-ns-big.jpg')
|
35
33
|
end
|
36
34
|
|
37
35
|
it 'image.alt を実行した場合' do
|
38
36
|
@doc.image.alt.should == '札幌入団記者会見でイエス・キリストのようなポーズを見せる中山'
|
39
37
|
end
|
38
|
+
|
39
|
+
it 'auther が無い場合' do
|
40
|
+
no_image_doc = Consadole::NikkanSports::Entry::parse(File.open(TEST_WITHOUT_AUTHER_FILE_NAME).read)
|
41
|
+
no_image_doc.auther.should be_false
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'image が無い場合' do
|
45
|
+
no_image_doc = Consadole::NikkanSports::Entry::parse(File.open(TEST_WITHOUT_IMAGE_FILE_NAME).read)
|
46
|
+
no_image_doc.image.should be_false
|
47
|
+
end
|
40
48
|
end
|
41
49
|
|
42
|
-
describe
|
50
|
+
describe Consadole::NikkanSports do
|
43
51
|
before do
|
44
|
-
@uris =
|
52
|
+
@uris = Consadole::NikkanSports.uris
|
45
53
|
end
|
46
54
|
|
47
55
|
it 'get を実行した場合 size が 50 であること' do
|
48
|
-
p @uris
|
49
56
|
@uris.should have(50).itmes
|
50
57
|
end
|
51
58
|
|