consadole_aggregator 0.0.0

data/.gitignore

@@ -0,0 +1,3 @@
+data/*
+config/*
+db/*

data/Rakefile

@@ -0,0 +1,32 @@
+require 'rubygems'
+require 'rake'
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "consadole_aggregator"
+    gemspec.summary = "Consadole Sapporo's Information Aggregator"
+    gemspec.email = "niku@niku.name"
+    gemspec.homepage = "http://github.com/niku/consadole_aggregator"
+    gemspec.authors = ["niku"]
+    gemspec.required_ruby_version = '~>1.9.1'
+    gemspec.add_development_dependency(['nokogiri','dm-core','dm-types','dm-validations'])
+    gemspec.add_development_dependency('rspec')
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.0.0

data/consadole_aggregator.gemspec

@@ -0,0 +1,182 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{consadole_aggregator}
+  s.version = "0.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["niku"]
+  s.date = %q{2009-12-21}
+  s.email = %q{niku@niku.name}
+  s.files = [
+    ".gitignore",
+     "Rakefile",
+     "VERSION",
+     "consadole_aggregator.gemspec",
+     "lib/consadole_aggregator.rb",
+     "lib/consadole_aggregator/live.rb",
+     "lib/consadole_aggregator/live/builder.rb",
+     "lib/consadole_aggregator/live/parser.rb",
+     "lib/consadole_aggregator/nikkan_sports.rb",
+     "lib/consadole_aggregator/nikkan_sports/builder.rb",
+     "lib/consadole_aggregator/nikkan_sports/entry.rb",
+     "lib/consadole_aggregator/nikkan_sports/parser.rb",
+     "lib/consadole_aggregator/result",
+     "spec/consadole_aggregator/nikkan_sports_spec.rb",
+     "spec/consadole_aggregator_spec.rb",
+     "spec/spec.opts",
+     "spec/spec_helper.rb",
+     "spec/timeline/s674.html",
+     "spec/timeline/s674.html.1",
+     "spec/timeline/s674.html.10",
+     "spec/timeline/s674.html.100",
+     "spec/timeline/s674.html.101",
+     "spec/timeline/s674.html.102",
+     "spec/timeline/s674.html.103",
+     "spec/timeline/s674.html.104",
+     "spec/timeline/s674.html.105",
+     "spec/timeline/s674.html.106",
+     "spec/timeline/s674.html.107",
+     "spec/timeline/s674.html.108",
+     "spec/timeline/s674.html.109",
+     "spec/timeline/s674.html.11",
+     "spec/timeline/s674.html.110",
+     "spec/timeline/s674.html.111",
+     "spec/timeline/s674.html.112",
+     "spec/timeline/s674.html.113",
+     "spec/timeline/s674.html.114",
+     "spec/timeline/s674.html.115",
+     "spec/timeline/s674.html.116",
+     "spec/timeline/s674.html.117",
+     "spec/timeline/s674.html.118",
+     "spec/timeline/s674.html.119",
+     "spec/timeline/s674.html.12",
+     "spec/timeline/s674.html.120",
+     "spec/timeline/s674.html.13",
+     "spec/timeline/s674.html.14",
+     "spec/timeline/s674.html.15",
+     "spec/timeline/s674.html.16",
+     "spec/timeline/s674.html.17",
+     "spec/timeline/s674.html.18",
+     "spec/timeline/s674.html.19",
+     "spec/timeline/s674.html.2",
+     "spec/timeline/s674.html.20",
+     "spec/timeline/s674.html.21",
+     "spec/timeline/s674.html.22",
+     "spec/timeline/s674.html.23",
+     "spec/timeline/s674.html.24",
+     "spec/timeline/s674.html.25",
+     "spec/timeline/s674.html.26",
+     "spec/timeline/s674.html.27",
+     "spec/timeline/s674.html.28",
+     "spec/timeline/s674.html.29",
+     "spec/timeline/s674.html.3",
+     "spec/timeline/s674.html.30",
+     "spec/timeline/s674.html.31",
+     "spec/timeline/s674.html.32",
+     "spec/timeline/s674.html.33",
+     "spec/timeline/s674.html.34",
+     "spec/timeline/s674.html.35",
+     "spec/timeline/s674.html.36",
+     "spec/timeline/s674.html.37",
+     "spec/timeline/s674.html.38",
+     "spec/timeline/s674.html.39",
+     "spec/timeline/s674.html.4",
+     "spec/timeline/s674.html.40",
+     "spec/timeline/s674.html.41",
+     "spec/timeline/s674.html.42",
+     "spec/timeline/s674.html.43",
+     "spec/timeline/s674.html.44",
+     "spec/timeline/s674.html.45",
+     "spec/timeline/s674.html.46",
+     "spec/timeline/s674.html.47",
+     "spec/timeline/s674.html.48",
+     "spec/timeline/s674.html.49",
+     "spec/timeline/s674.html.5",
+     "spec/timeline/s674.html.50",
+     "spec/timeline/s674.html.51",
+     "spec/timeline/s674.html.52",
+     "spec/timeline/s674.html.53",
+     "spec/timeline/s674.html.54",
+     "spec/timeline/s674.html.55",
+     "spec/timeline/s674.html.56",
+     "spec/timeline/s674.html.57",
+     "spec/timeline/s674.html.58",
+     "spec/timeline/s674.html.59",
+     "spec/timeline/s674.html.6",
+     "spec/timeline/s674.html.60",
+     "spec/timeline/s674.html.61",
+     "spec/timeline/s674.html.62",
+     "spec/timeline/s674.html.63",
+     "spec/timeline/s674.html.64",
+     "spec/timeline/s674.html.65",
+     "spec/timeline/s674.html.66",
+     "spec/timeline/s674.html.67",
+     "spec/timeline/s674.html.68",
+     "spec/timeline/s674.html.69",
+     "spec/timeline/s674.html.7",
+     "spec/timeline/s674.html.70",
+     "spec/timeline/s674.html.71",
+     "spec/timeline/s674.html.72",
+     "spec/timeline/s674.html.73",
+     "spec/timeline/s674.html.74",
+     "spec/timeline/s674.html.75",
+     "spec/timeline/s674.html.76",
+     "spec/timeline/s674.html.77",
+     "spec/timeline/s674.html.78",
+     "spec/timeline/s674.html.79",
+     "spec/timeline/s674.html.8",
+     "spec/timeline/s674.html.80",
+     "spec/timeline/s674.html.81",
+     "spec/timeline/s674.html.82",
+     "spec/timeline/s674.html.83",
+     "spec/timeline/s674.html.84",
+     "spec/timeline/s674.html.85",
+     "spec/timeline/s674.html.86",
+     "spec/timeline/s674.html.87",
+     "spec/timeline/s674.html.88",
+     "spec/timeline/s674.html.89",
+     "spec/timeline/s674.html.9",
+     "spec/timeline/s674.html.90",
+     "spec/timeline/s674.html.91",
+     "spec/timeline/s674.html.92",
+     "spec/timeline/s674.html.93",
+     "spec/timeline/s674.html.94",
+     "spec/timeline/s674.html.95",
+     "spec/timeline/s674.html.96",
+     "spec/timeline/s674.html.97",
+     "spec/timeline/s674.html.98",
+     "spec/timeline/s674.html.99"
+  ]
+  s.homepage = %q{http://github.com/niku/consadole_aggregator}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.required_ruby_version = Gem::Requirement.new("~> 1.9.1")
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Consadole Sapporo's Information Aggregator}
+  s.test_files = [
+    "spec/consadole_aggregator/nikkan_sports_spec.rb",
+     "spec/consadole_aggregator_spec.rb",
+     "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<["nokogiri", "dm-core", "dm-types", "dm-validations"]>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, [">= 0"])
+    else
+      s.add_dependency(%q<["nokogiri", "dm-core", "dm-types", "dm-validations"]>, [">= 0"])
+      s.add_dependency(%q<rspec>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<["nokogiri", "dm-core", "dm-types", "dm-validations"]>, [">= 0"])
+    s.add_dependency(%q<rspec>, [">= 0"])
+  end
+end
+

data/lib/consadole_aggregator.rb

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+require 'logger'
+
+require 'consadole_aggregator/nikkan_sports'
+require 'consadole_aggregator/live'
+
+module ConsadoleAggregator
+  def self.log
+    @@log ||= Logger.new('log/application.log')
+  end
+
+  Image = Struct.new :src, :alt do
+    def to_html
+      %Q{<image src="#{self.src}" alt="#{self.alt}">}
+    end
+  end
+end
+
+if __FILE__ == $0
+  DataMapper.setup(:default, 'sqlite3:db/test.sqlite3')
+  DataMapper.auto_migrate!
+
+  ConsadoleAggregator::NikkanSports.save
+  print ConsadoleAggregator::NikkanSports.rss
+end

data/lib/consadole_aggregator/live/builder.rb

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+require 'twitter'
+
+module ConsadoleAggregator
+  module Live
+    module Builder
+      class Twit
+        def initialize
+          Process.daemon(true, false)
+          @client = Twitter::Client.from_config('config/account.yaml')
+          @posted = []
+        end
+
+        def post
+          new_line.each do |line|
+            @client.status(:post, "#{line.time} #{line.post} #consadole")
+            @posted << line
+          end
+        end
+
+        def reserve start_time
+          start = Time.parse start_time
+          ConsadoleAggregator::log.info("#{start.to_time}くらいにはじめる予定")
+          sleep start - Time.now
+
+          live
+        end
+
+        def live
+          # サッカーはハーフタイム含めて120分位あれば終わる
+          120.times do
+            post
+            sleep 60
+          end
+        end
+
+        private
+        def new_line
+          ConsadoleAggregator::Live::Parser.parse - @posted
+        end
+      end
+    end
+  end
+end

data/lib/consadole_aggregator/live/parser.rb

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+require 'net/http'
+require 'kconv'
+
+module ConsadoleAggregator
+  module Live
+    module Parser
+      BASE_URI = URI.parse('http://www.consadole-sapporo.jp/view/s674.html')
+      TimeLine = Struct.new :time, :post do
+        def initialize line
+          tmp = line.split
+          self['time'] = tmp[0]
+          self['post'] = tmp[1]
+        end
+      end
+
+      def Parser.parse
+        response = Net::HTTP.get(BASE_URI).force_encoding("Shift_JIS")
+        doc = Nokogiri::HTML.parse(response.toutf8, nil, 'UTF-8')
+        doc.search('hr + p').last.inner_html.gsub(/<br>/,'').split("\n").map{ |line|
+          TimeLine.new line unless /&lt;前半&gt;|&lt;後半&gt;/ =~ line
+        }.compact.reverse
+      end
+    end
+  end
+end

data/lib/consadole_aggregator/nikkan_sports.rb

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+require 'net/http'
+require 'kconv'
+
+require 'rubygems'
+require 'nokogiri'
+
+require 'consadole_aggregator/nikkan_sports/entry'
+require 'consadole_aggregator/nikkan_sports/parser'
+require 'consadole_aggregator/nikkan_sports/builder'
+
+module ConsadoleAggregator
+  module NikkanSports
+    def NikkanSports.list
+      doc = Nokogiri::HTML.parse(Net::HTTP.get(Parser::List::BASE_URI), nil, 'UTF-8')
+      Parser::List.parse doc
+    end
+
+    def NikkanSports.save
+      NikkanSports.list.map(&:uri).each do |uri|
+        doc = Nokogiri::HTML.parse(Net::HTTP.get(uri), nil, 'UTF-8').at('div#news')
+        entry = Parser::Entry.parse doc, uri
+        entry.save # DataMapper だと 一意制約違反でも例外にならない。ただ挿入できないだけ。
+      end
+    end
+
+    def NikkanSports.rss
+      entries = Entry.all(:order => [:post_date.desc], :limit => 20)
+
+      Builder::Rss.build entries do |item, entry|
+        item.link = entry.uri.to_s.toutf8
+        item.title = entry.title.toutf8
+        item.summary = "<div>#{entry.image ? entry.image.to_html + entry.text : entry.text}</div>".toutf8
+        item.date = entry.post_date
+      end
+    end
+  end
+end

data/lib/consadole_aggregator/nikkan_sports/builder.rb

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+require 'rss'
+
+module ConsadoleAggregator
+  module NikkanSports
+    module Builder
+      module Rss
+        def Rss.build entries
+          RSS::Maker.make('atom') do |maker|
+            maker.channel.about = 'http://sirase.ru/consadole/nikkansports_consadole.rdf'
+            maker.channel.title = 'コンサドーレ札幌:nikkansports.com(非公式RSS版)'
+            maker.channel.description = 'コンサドーレ札幌:nikkansports.comのニュース一覧(非公式RSS版)'
+            maker.channel.link = 'http://www.nikkansports.com/soccer/jleague/team/co-team.html'
+            maker.channel.author = 'ConsadoleAggregator'
+            maker.channel.date = Time.now
+
+            maker.items.do_sort = true
+
+            entries.each do |entry|
+              maker.items.new_item do |item|
+                yield item, entry
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/consadole_aggregator/nikkan_sports/entry.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'dm-core'
+require 'dm-types'
+require 'dm-validations'
+
+module ConsadoleAggregator
+  module NikkanSports
+    class Entry
+      include DataMapper::Resource
+
+      property :id, Serial
+      property :uri, URI, :unique => true
+      property :title, String
+      property :text, Text
+      property :post_date, Time
+      property :image, String
+    end
+  end
+end

data/lib/consadole_aggregator/nikkan_sports/parser.rb

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+module ConsadoleAggregator
+  module NikkanSports
+    module Parser
+      module List
+        BASE_URI = URI.parse('http://www.nikkansports.com/soccer/jleague/team/co-team.html')
+
+        Element = Struct.new :uri, :title, :post_date, :has_image? do
+          def initialize element
+            self['uri'] = URI.parse(element.at('a')['href'])
+            self['title'] = element.at('a').text
+            self['post_date'] = Time.strptime(element.at('span').text.match(/\d+日\d+:\d+/).to_s, '%d日%H:%M')
+            self['has_image?'] = !element.at('img').nil?
+          end
+        end
+
+        def List.parse doc
+          doc.search('div#topNewsInner ul.newsList01 li').map{|element| Element.new element}
+        end
+      end
+
+      module Entry
+        IMAGE_BASE_URI = URI.parse('http://cache2.nipc.jp/soccer/news/')
+
+        def Entry.parse doc, uri = nil
+          entry = NikkanSports::Entry.new
+          entry.uri = uri if uri
+          entry.title = doc.at('h1').text
+          entry.text = doc.search('p')[0...-1].to_s
+          entry.post_date = Time.strptime(doc.search('p').last.text.match(/\d+年\d+月\d+日\d+時\d+分/).to_s, '%Y年%m月%d日%H時%M分')
+          entry.image = Image.new(IMAGE_BASE_URI + image['src'].sub(/\.jpg$/, '-big.jpg'), image['alt']) unless doc.at('img').nil?
+          return entry
+        end
+      end
+    end
+  end
+end