consadole_aggregator 0.1.9 → 0.2.0

data/lib/consadole_aggregator/live.rb

@@ -1,88 +1,95 @@
 # -*- coding: utf-8 -*-
-require 'logger'
 require 'uri'
+require 'net/http'
 require 'kconv'
 require 'nokogiri'
-require 'net/http'
-require_relative 'live/timeline.rb'
+require 'eventmachine'
 
 module ConsadoleAggregator
-  module Live
-
-    def self.reserve reservation_time=nil, opt ={}
-      Live.new(reservation_time, opt)
-    end
-
-    class Live
-      include Aggregatable
-      attr_reader :reservation_time, :posted, :times, :wait_sec
+  class Live
+    class Runner
+      attr_writer :length_of_a_game, :interval, :daemonize
 
-      def initialize reservation_time=nil, opt ={}
-        @reservation_time = reservation_time
-        @posted = []
-        @wait_sec = opt[:wait_sec] || 30
-        @times = opt[:times] || (60/@wait_sec)*120 # サッカーは120分あれば終わる
-        @logger = opt[:logger] || Logger.new(File.expand_path(File.dirname(__FILE__) + '/../../log/live.log'))
+      def initialize starting_time
+        @starting_time = starting_time
+        @length_of_a_game = 120 * 60
+        @interval = 60
+        @daemonize = true
       end
 
-      def execute &block
-        be_daemonize
-        wait_initial
-        @logger.info 'start of loop'
-        @times.times do |i|
-          @logger.debug "#{i} times"
-          update &block rescue @logger.error $!
-          wait_interval
-        end
-        @logger.info 'end of loop'
-      end
+      def run &block
+        Process.daemon if @daemonize
 
-      def wait_initial
-        return unless @reservation_time
-        diff_sec = @reservation_time - Time.now
-        wait_sec = diff_sec > 0 ? diff_sec : 0
-        @logger.info "initial wait #{wait_sec} seconds"
-        sleep wait_sec
-      end
+        starting = @starting_time - Time.now
+        ending = starting + @length_of_a_game
+        live = Live.new
 
-      def wait_interval
-        sleep @wait_sec
-      end
+        EM.run do
+          EM.add_timer(starting) do
+            EM.next_tick do
+              EM.add_periodic_timer(@interval) do
+                live.update &block
+              end
+            end
+          end
 
-      def update
-        new_timeline = ConsadoleAggregator::Live.parse - @posted
-        new_timeline.each do |timeline|
-          @logger.debug timeline
-          yield timeline if block_given?
-          @posted << timeline
+          EM.add_timer(ending) do
+            EM.stop
+          end
         end
       end
+    end
 
-      def self.get_resource
-        ->{ Net::HTTP.get(URI.parse('http://www.consadole-sapporo.jp/view/s674.html')).toutf8 }
-      end
+    def self.run starting_time, opt = {}
+      runner = Runner.new starting_time
+      runner.length_of_a_game = opt[:length_of_a_game] if opt[:length_of_a_game]
+      runner.interval = opt[:interval] if opt[:interval]
+      runner.daemonize = opt[:daemonize] unless opt[:daemonize].nil?
+      runner.run &opt[:writer]
+    end
 
-      def self.parse_list
-        ->(list){
-          live_block = Nokogiri::HTML::parse(list)
-            .search("hr + p")
-            .last
-            .inner_html
-          lines = live_block
-            .split(/<br>|\n/)
-            .delete_if{ |line| line.empty? || line =~ /&lt;前|後半&gt;/ }
-          lines.map{ |line| line.sub(/　+$/, "") }.reverse
-        }
-      end
+    attr_reader :posted
 
-      def self.parse_article
-        ->(article){ Timeline.parse article }
-      end
+    def initialize
+      @posted = []
+    end
 
-      private
-      def be_daemonize
-        Process.daemon
-      end
+    def update
+      self.class
+        .fetch
+        .reject(&@posted.method(:include?))
+        .each_with_object(@posted) { |post, posted|
+        begin
+          yield({ title: post }) if block_given?
+          posted << post
+          ConsadoleAggregator.logger.info post
+        rescue
+          ConsadoleAggregator.logger.error $!
+        end
+      }
+    end
+
+    private
+    def self.parse document
+      Nokogiri
+        .parse(document)
+        .search('p:last-of-type')
+        .children
+        .grep(Nokogiri::XML::Text)
+        .map(&:text)
+        .map(&:toutf8)
+        .map { |t| t.tr('　', ' ') }
+        .map(&:strip)
+        .reject(&/<前|後半>/.method(:match))
+        .reverse
+    end
+
+    def self.fetch
+      uri = URI.parse('http://www.consadole-sapporo.jp/view/s674.html')
+      doc = Net::HTTP.get(uri).force_encoding('SJIS')
+      parse(doc)
+    rescue
+      []
     end
   end
 end

data/lib/consadole_aggregator/news.rb

@@ -1,117 +1,182 @@
 # -*- coding: utf-8 -*-
-require 'logger'
 require 'rss'
-require 'uri'
-require 'kconv'
-require 'net/http'
-require 'net/https'
+require 'pstore'
 require 'nokogiri'
+require 'httpclient'
 
 module ConsadoleAggregator
   module News
-    def self.get_resource(url_path)
-      Net::HTTP.get(URI.parse(url_path)).toutf8
+    class Register
+      def initialize
+        @sites = []
+      end
+
+      def name name
+        builder = SiteBuilder.new name
+        yield builder if block_given?
+        @sites << builder.build
+      end
+
+      def register!
+        News.const_set(:Sites, []) unless News.const_defined?(:Sites)
+        News.const_get(:Sites).concat @sites
+      end
     end
 
-    def self.trace(url_path, limit=10)
-      raise ArgumentError, 'http redirect too deep' if limit == 0
+    class SiteBuilder
+      def initialize name
+        @name = name
+      end
 
-      case response = Net::HTTP.get_response(URI.parse(url_path))
-      when Net::HTTPSuccess     then url_path
-      when Net::HTTPRedirection then trace(response['Location'], limit - 1)
-      else
-        response.error!
+      def resource &block; @resource = block; end
+      def parse_list &block; @parse_list = block; end
+      def filter_article &block; @filter_article = block; end
+      def parse_article &block; @parse_article = block; end
+
+      def build
+        site = Site.new @name
+        site.resource = @resource if @resource
+        site.list_parser = @parse_list if @parse_list
+        site.article_filter = @filter_article if @filter_article
+        site.article_parser = @parse_article if @parse_article
+        site
       end
     end
 
-    {
-      Nikkansports:
-      [
-       ->{ get_resource('http://www.nikkansports.com/rss/soccer/jleague/consadole.rdf') },
-       ->(list){ RSS::Parser.parse(list, false).items.map{ |e| { url:e.link, title:e.title } }.reverse },
-       ->(article){ article }
-      ],
-      Hochiyomiuri:
-      [
-       ->{ get_resource('http://hochi.yomiuri.co.jp/hokkaido/soccer/index.htm') },
-       ->(list){ Nokogiri::HTML(list).search('div.list1 > ul > li a').reverse },
-       ->(article){ { url:"http://hochi.yomiuri.co.jp#{article['href']}", title:article.text } if article.text =~ /…札幌$/ }
-      ],
-      Asahi:
-      [
-       ->{ get_resource('http://mytown.asahi.com/hokkaido/newslist.php?d_id=0100019') },
-       ->(list){ Nokogiri::HTML(list).search('ul.list > li a').reverse },
-       ->(article){ { url:"http://mytown.asahi.com/hokkaido/#{article['href']}", title:article.text } }
-      ],
-      Forzaconsadole:
-      [
-       ->{ get_resource('http://www.hokkaido-np.co.jp/news/e_index/?g=consadole') },
-       ->(list){ Nokogiri::HTML(list).search('ul.iSwBox > li > a').reverse },
-       ->(article){ { url:article['href'], title:article.text } }
-      ],
-      Consaburn:
-      [
-       ->{ get_resource('http://www.hokkaido-np.co.jp/cont/consa-burn/index.html') },
-       ->(list){ Nokogiri::HTML(list).search('ul#news_list > li > a').reverse },
-       ->(article){ { url:article['href'], title:article.text } }
-      ],
-      Consaclub:
-      [
-       ->{ get_resource('http://www.hokkaido-np.co.jp/cont/consa-club/index.html') },
-       ->(list){ Nokogiri::HTML(list).search('ul#news_list > li > a').reverse },
-       ->(article){ { url:article['href'], title:article.text } }
-      ],
-      Consadolenews:
-      [
-       ->{ get_resource('http://www.consadole-sapporo.jp/news/atom.xml') },
-       ->(list){ Nokogiri::XML(list).search('entry').reverse },
-       ->(article){ { title:article.at('title').text, url:article.at('link')['href'] } }
-      ],
-      Consadolephotos:
-      [
-       ->{ get_resource('http://www.consadole-sapporo.jp/') },
-       ->(list){ Nokogiri::HTML(list).search('div.anythingSlider img').reverse },
-       ->(article){ { url:article['src'], title:article['alt'] } }
-      ],
-      Jsgoalnews:
-      [
-       ->{ get_resource('http://feeds.feedburner.com/jsgoal/jsgoal?format=xml') },
-       ->(list){
-         RSS::Parser.parse(list, false).items.each_with_object([]){ |e, memo|
-           memo << { url:trace(e.link), title:e.title } if e.title.include?('札幌')
-         }.reverse },
-       ->(article){ article }
-      ],
-      Jsgoalphotos:
-      [
-       ->{ get_resource('http://feeds.feedburner.com/jsgoal/photo?format=xml') },
-       ->(list){
-         RSS::Parser.parse(list, false).items.each_with_object([]){ |e, memo|
-           memo << { url:trace(e.link), title:e.title } if e.title.include?('札幌')
-         }.reverse },
-       ->(article){ article }
-      ],
-      Clubconsadole:
-      [
-       ->{
-         uri = URI.parse('https://www.finn.ne.jp/user.cgi?fanclub_id=67&actmode=NewsArticleSummary')
-         https = Net::HTTP.new(uri.host, uri.port)
-         https.use_ssl = true
-         https.verify_mode = OpenSSL::SSL::VERIFY_NONE
-         https.start { https.get(uri.request_uri).body.toutf8 }
-       },
-       ->(list){ Nokogiri::HTML(list).search('dl dd').reverse },
-       ->(article){ { url:'https://www.finn.ne.jp/' + article.at('a')['href'], title: article.text } }
-      ],
-    }.each do |k,v|
-      klass = Class.new do
-        include Aggregatable
-        @get_resource, @parse_list, @parse_article = *v
-        def initialize logger=nil
-          @logger = logger || Logger.new(File.expand_path(File.dirname(__FILE__) + '/../../log/news.log'))
+    class Site
+      class << self
+        STORE_NAME = 'updated.pstore'
+        def store
+          @store ||= PStore.new(File.join(ConsadoleAggregator.root_dir, STORE_NAME))
         end
       end
-      const_set(k, klass)
+
+      STORAGE_ELEMENT_SIZE = 200
+
+      attr_writer :resource, :list_parser, :article_filter, :article_parser
+      attr_reader :name
+
+      def initialize name
+        @name = name
+      end
+
+      def updated
+        @updated ||= self.class.store.transaction { |s| s.fetch(name, []) }
+      end
+
+      def update
+        resource = @resource.call
+        list = @list_parser.call(resource).to_a
+        list.select! &@article_filter if @article_filter
+        list.map! &@article_parser if @article_parser
+        list.reject! { |article| updated.include?(article) }
+        list.each { |article|
+          begin
+            yield article if block_given?
+            add_updated article
+            ConsadoleAggregator.logger.info article
+          rescue
+            ConsadoleAggregator.logger.error $!
+          end
+        }
+      rescue
+        ConsadoleAggregator.logger.error $!
+      end
+
+      private
+      def add_updated article
+        @updated << article
+        self.class.store.transaction { |s| s[name] = @updated.last(STORAGE_ELEMENT_SIZE) }
+      end
+    end
+
+    def self.run &block
+      Sites.each { |site| site.update &block }
+    end
+
+    def self.register!
+      register = Register.new
+      yield register if block_given?
+      register.register!
+    end
+
+    register! do |sites|
+      sites.name(:nikkansports) do |site|
+        site.resource { HTTPClient.new.get_content('http://www.nikkansports.com/rss/soccer/jleague/consadole.rdf').force_encoding('UTF-8') }
+        site.parse_list { |list| RSS::Parser.parse(list, false).items.reverse }
+        site.parse_article { |article| { url: article.link, title: article.title } }
+      end
+
+      sites.name(:hochiyomiuri) do |site|
+        site.resource { HTTPClient.new.get_content('http://hochi.yomiuri.co.jp/hokkaido/soccer/index.htm').force_encoding('UTF-8') }
+        site.parse_list { |list| Nokogiri::HTML(list).search('div.list1 > ul > li a').reverse }
+        site.filter_article { |article| article.text =~ /札幌|コンサ/ }
+        site.parse_article { |article| { url:"http://hochi.yomiuri.co.jp#{article['href']}", title:article.text } }
+      end
+
+      sites.name(:asahi) do |site|
+        site.resource { HTTPClient.new.get_content('http://mytown.asahi.com/hokkaido/newslist.php?d_id=0100019').encode('UTF-8') }
+        site.parse_list { |list| Nokogiri::HTML(list).search('ul.list > li a').reverse }
+        site.parse_article { |article| { url: "http://mytown.asahi.com/hokkaido/#{article['href']}", title:article.text } }
+      end
+
+      sites.name(:forzaconsadole) do |site|
+        site.resource { HTTPClient.new.get_content('http://www.hokkaido-np.co.jp/news/e_index/?g=consadole').encode('UTF-8') }
+        site.parse_list { |list| Nokogiri::HTML(list).search('ul.iSwBox > li > a').reverse }
+        site.parse_article { |article| { url: article['href'], title: article.text } }
+      end
+
+      sites.name(:consaburn) do |site|
+        site.resource { HTTPClient.new.get_content('http://www.hokkaido-np.co.jp/cont/consa-burn/index.html').encode('UTF-8') }
+        site.parse_list { |list| Nokogiri::HTML(list).search('ul#news_list > li > a').reverse }
+        site.parse_article { |article| { url: article['href'], title: article.text } }
+      end
+
+      sites.name(:consaclub) do |site|
+        site.resource { HTTPClient.new.get_content('http://www.hokkaido-np.co.jp/cont/consa-club/index.html').encode('UTF-8') }
+        site.parse_list { |list| Nokogiri::HTML(list).search('ul#news_list > li > a').reverse }
+        site.parse_article { |article| { url: article['href'], title: article.text } }
+      end
+
+      sites.name(:consadolenews) do |site|
+        site.resource { HTTPClient.new.get_content('http://www.consadole-sapporo.jp/news/atom.xml').force_encoding('UTF-8') }
+        site.parse_list { |list| Nokogiri::XML(list).search('entry').reverse }
+        site.parse_article { |article| { title: article.at('title').text, url: article.at('link')['href'] } }
+      end
+
+      sites.name(:consadolephotos) do |site|
+        site.resource { HTTPClient.new.get_content('http://www.consadole-sapporo.jp/').force_encoding('UTF-8') }
+        site.parse_list { |list| Nokogiri::HTML(list).search('div.anythingSlider img').reverse }
+        site.parse_article { |article| { url: article['src'], title: article['alt'] } }
+      end
+
+      sites.name(:jsgoalnews) do |site|
+        site.resource { HTTPClient.new.get_content('http://feeds.feedburner.com/jsgoal/jsgoal?format=xml').encode('UTF-8') }
+        site.parse_list { |list| RSS::Parser.parse(list, false).items.reverse }
+        site.filter_article { |article| article.title =~ /札幌/ }
+        site.parse_article { |article|
+          c = HTTPClient.new.get(article.link)
+          c = HTTPClient.new.get(c.header['location'].first) while c.status == 301
+          { url: c.header['location'].first, title: article.title }
+        }
+      end
+
+      sites.name(:jsgoalphotos) do |site|
+        site.resource { HTTPClient.new.get_content('http://feeds.feedburner.com/jsgoal/photo?format=xml').encode('UTF-8') }
+        site.parse_list { |list| RSS::Parser.parse(list, false).items.reverse }
+        site.filter_article { |article| article.title =~ /札幌/ }
+        site.parse_article { |article|
+          c = HTTPClient.new.get(article.link)
+          c = HTTPClient.new.get(c.header['location'].first) while c.status == 301
+          { url: c.header['location'].first, title: article.title }
+        }
+      end
+
+      sites.name(:clubconsadole) do |site|
+        site.resource { HTTPClient.new.get_content('http://club-consadole.jp/news/index.php?page=1').encode('UTF-8') }
+        site.parse_list { |list| Nokogiri::HTML(list).search('li.news a').reverse }
+        site.parse_article { |article| { url: article['href'], title: article.text.strip } }
+      end
     end
   end
 end

data/lib/consadole_aggregator/version.rb

@@ -0,0 +1,3 @@
+module ConsadoleAggregator
+  VERSION = "0.2.0"
+end

data/lib/consadole_aggregator.rb

@@ -1,8 +1,30 @@
+require "consadole_aggregator/version"
+require 'consadole_aggregator/helper.rb'
+require 'consadole_aggregator/live.rb'
+require 'consadole_aggregator/news.rb'
+
 require 'logger'
-require_relative 'consadole_aggregator/helper.rb'
-require_relative 'consadole_aggregator/aggregatable.rb'
-require_relative 'consadole_aggregator/live.rb'
-require_relative 'consadole_aggregator/news.rb'
+require 'fileutils'
 
 module ConsadoleAggregator
+  class << self
+    DEFAULT_ROOT_DIR = File.expand_path('~/.consadole_aggregator')
+    DEFAULT_LOG_NAME = 'consadole_aggregator.log'
+
+    def root_dir
+      return @root_dir if @root_dir
+      FileUtils.mkdir_p DEFAULT_ROOT_DIR unless File.exist? DEFAULT_ROOT_DIR
+      @root_dir = DEFAULT_ROOT_DIR
+    end
+
+    def root_dir= root_dir
+      FileUtil.mkdir_p root_dir unless File.exist? root_dir
+      @logger &&= Logger.new(File.join(root_dir, DEFAULT_LOG_NAME))
+      @root_dir = root_dir
+    end
+
+    def logger
+      @logger ||= Logger.new(File.join(root_dir, DEFAULT_LOG_NAME))
+    end
+  end
 end

data/spec/consadole_aggregator/helper_spec.rb

@@ -1,27 +1,32 @@
 # -*- coding: utf-8 -*-
-require_relative '../spec_helper'
+require 'spec_helper'
 
 describe ConsadoleAggregator::Helper do
-  describe :concatenate do
+  describe '#truncate_for_twitter' do
     context 'with argument 140 chars' do
-      subject { ConsadoleAggregator::Helper.concat('い' * 140) }
+      subject { ConsadoleAggregator::Helper.truncate_for_twitter('い' * 140) }
       it { should have(140).item }
       it { should == 'い' * 140 }
     end
     context 'with argument 140 chars and 18 chars url' do
-      subject { ConsadoleAggregator::Helper.concat('ろ' * 140, url:'http://example.jp/') }
-      it { should have(140).item }
+      subject { ConsadoleAggregator::Helper.truncate_for_twitter('ろ' * 140, url:'http://example.jp/') }
+      it { should have(138).item }
       it { should be_end_with 'ろ... http://example.jp/' }
     end
     context 'with argument 140 chars and 10 chars hashtag' do
-      subject { ConsadoleAggregator::Helper.concat('は' * 140, hashtag:'#consadole') }
+      subject { ConsadoleAggregator::Helper.truncate_for_twitter('は' * 140, hashtag:'#consadole') }
       it { should have(140).item }
       it { should be_end_with 'は... #consadole' }
     end
     context 'with argument 140 chars and 18 chars url and 10 chars hashtag' do
-      subject { ConsadoleAggregator::Helper.concat('に' * 140, url:'http://example.jp/', hashtag:'#consadole') }
-      it { should have(140).item }
+      subject { ConsadoleAggregator::Helper.truncate_for_twitter('に' * 140, url:'http://example.jp/', hashtag:'#consadole') }
+      it { should have(138).item }
       it { should be_end_with 'に... http://example.jp/ #consadole' }
     end
+    context 'with argument 140 chars and 29 chars url and 10 chars hashtag' do
+      subject { ConsadoleAggregator::Helper.truncate_for_twitter('に' * 140, url:'http://example.jp/foo/bar/baz', hashtag:'#consadole') }
+      it { should have(149).item }
+      it { should be_end_with 'に... http://example.jp/foo/bar/baz #consadole' }
+    end
   end
 end