crown 0.1.0 → 0.1.1

data/lib/crown/hatena/bookmark/entrylist.rb

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+# --------------------------------------------------------------------------- #
+#
+#  hatena/bookmark/entrylist.rb
+#
+#  Copyright (c) 2008 - 2012, clown.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    - Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    - Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#    - No names of its contributors may be used to endorse or promote
+#      products derived from this software without specific prior written
+#      permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+#  TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# --------------------------------------------------------------------------- #
+module Crown
+    module Hatena
+        class Bookmark
+            require 'crown/hatena/bookmark/urilist'
+            require 'crown/hatena/bookmark/entry'
+            
+            # --------------------------------------------------------------- #
+            #
+            #  EntryList
+            #
+            #  はてなブックマークの新着エントリー，または人気エントリー一覧
+            #  の各エントリー情報を取得するクラス．
+            #
+            # --------------------------------------------------------------- #
+            class EntryList < URIList
+                attr_accessor :interval
+                
+                # ----------------------------------------------------------- #
+                #  initialize
+                # ----------------------------------------------------------- #
+                def initialize(options = {})
+                    super
+                    @interval = 30
+                end
+                
+                # ----------------------------------------------------------- #
+                #  EntryList.start
+                # ----------------------------------------------------------- #
+                def EntryList.start(options = {}, &block)
+                    Crown::Hatena::Bookmark::EntryList.new(options).start(&block)
+                end
+                
+                # ----------------------------------------------------------- #
+                #  get
+                # ----------------------------------------------------------- #
+                def get()
+                    return get_summary(super()) if (!block_given?)
+                    while (more?)
+                        super().each { |entry|
+                            result = Crown::Hatena::Bookmark::Entry.new.summary(entry.uri,
+                                { :proxy_address => proxy_address(), :proxy_port => proxy_port() })
+                            yield result if (result != nil)
+                            sleep(@interval)
+                        }
+                    end
+                end
+                
+            private
+                # ----------------------------------------------------------- #
+                #  get_summary
+                # ----------------------------------------------------------- #
+                def get_summary(entries)
+                    dest = Array.new
+                    entries.each { |entry|
+                        result = Crown::Hatena::Bookmark::Entry.new.summary(entry.uri,
+                                { :proxy_address => proxy_address(), :proxy_port => proxy_port() })
+                        dest.add(result) if (result != nil)
+                        sleep(@interval)
+                    }
+                    return dest
+                end
+            end # EntryList
+        end # Bookmark
+    end # Hatena
+end # Crown

data/lib/crown/hatena/bookmark/urilist.rb

@@ -0,0 +1,349 @@
+# -*- coding: utf-8 -*-
+# --------------------------------------------------------------------------- #
+#
+#  hatena/bookmark/urilist.rb
+#
+#  Copyright (c) 2008 - 2012, clown.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    - Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    - Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#    - No names of its contributors may be used to endorse or promote
+#      products derived from this software without specific prior written
+#      permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+#  TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# --------------------------------------------------------------------------- #
+module Crown
+    module Hatena
+        class Bookmark
+            # --------------------------------------------------------------- #
+            #
+            #  URIList
+            #
+            #  指定した Web サイトのはてなブックマーク一覧から順に URL を
+            #  取得するクラス．2 ページ目以降の結果は，html のヘッダ情報の
+            #  next 属性からページの URL を推測して取得する．
+            #
+            # --------------------------------------------------------------- #
+            class URIList
+                require 'cgi'
+                require 'uri'
+                require 'date'
+                require 'crown/http-wrapper'
+                require 'rubygems'
+                require 'nokogiri'
+                
+                # ----------------------------------------------------------- #
+                #  structures
+                # ----------------------------------------------------------- #
+                Response = Struct.new(:uri, :title, :date)
+                
+                # ----------------------------------------------------------- #
+                #  variables
+                # ----------------------------------------------------------- #
+                @@valid_types      = [ :hotentry, :entrylist, :video, :asin ]
+                @@valid_categories = [ :general, :social, :economics, :life, :entertainment, :knowledge, :it, :game, :fun, :news ]
+                @@valid_options    = [ :uri, :sort, :threshold, :offset, :src ]
+                @@valid_sorts      = [ :eid, :hot, :count ]
+                @@valid_sources    = [ :youtube, :nicovideo, :ugomemo ]
+                
+                # ----------------------------------------------------------- #
+                #  initialize
+                # ----------------------------------------------------------- #
+                def initialize(options = {})
+                    proxy_addr = nil
+                    proxy_port = nil
+                    if (options.class == Hash)
+                        proxy_addr = options[:proxy_address] if (options.has_key?(:proxy_address))
+                        proxy_port = options[:proxy_port] if (options.has_key?(:proxy_port))
+                    end
+                    @session = Crown::HTTPWrapper.new('b.hatena.ne.jp', 80, proxy_addr, proxy_port)
+                    @basename = basename(options)
+                    @path = String.new(@basename)
+                end
+                
+                # ----------------------------------------------------------- #
+                #  URIList.start
+                # ----------------------------------------------------------- #
+                def URIList.start(options = {}, &block)
+                    return URIList.new(options).start(&block)
+                end
+                
+                # ----------------------------------------------------------- #
+                #  start
+                # ----------------------------------------------------------- #
+                def start()
+                    @session.start if (!@session.active?)
+                    if (block_given?)
+                        yield self
+                        @session.finish if (@session.active?)
+                    end
+                    return self
+                end
+                
+                # ----------------------------------------------------------- #
+                #  finish
+                # ----------------------------------------------------------- #
+                def finish()
+                    @session.finish if (@session.active?)
+                end
+                
+                # ----------------------------------------------------------- #
+                #  reset
+                # ----------------------------------------------------------- #
+                def reset()
+                    @path = String.new(@basename)
+                    @session.finish if (@session.active?)
+                    return self
+                end
+                
+                # ----------------------------------------------------------- #
+                #
+                #  more?
+                #
+                #  まだ取得できる URL が存在するかどうかを判定する．more?()
+                #  は html の next 属性から次のページが推測できているか
+                #  どうかで判定している．
+                #
+                # ----------------------------------------------------------- #
+                def more?()
+                    return @path != nil
+                end
+                
+                # ----------------------------------------------------------- #
+                #  get
+                # ----------------------------------------------------------- #
+                def get()
+                    return get_block() if (!block_given?)
+                    while (more?)
+                        get_block().each { |uri|
+                            yield uri
+                        }
+                    end
+                end
+                
+                # ----------------------------------------------------------- #
+                #  proxy_address
+                # ----------------------------------------------------------- #
+                def proxy_address
+                    return @session.proxy_address
+                end
+                
+                # ----------------------------------------------------------- #
+                #  proxy_port
+                # ----------------------------------------------------------- #
+                def proxy_port
+                    return @session.proxy_port
+                end
+                
+            private
+                # ----------------------------------------------------------- #
+                #
+                #  get_block
+                #
+                #  はてなブックマークの新着エントリーに掲載された URL の
+                #  各種情報（URL, タイトル，日付）を取得する．
+                #
+                # ----------------------------------------------------------- #
+                def get_block()
+                    result = Array.new
+                    
+                    return result if (@path == nil)
+                    
+                    response = @session.get(@path)
+                    return result if (response == nil || response.code.to_i != 200)
+                    @path = path(@path, response.body)
+                    
+                    html = Nokogiri::HTML(response.body)
+                    
+                    # Amazon 商品のブックマーク情報のみ構成が異なるので別メソッドで処理する．
+                    contents = html.xpath('//ul[@class="hotentry hotasin show_detail"]/li')
+                    return get_asin_block(contents) if (!contents.empty?)
+                    
+                    contents = html.xpath('//ul[@class="hotentry"]/li')
+                    contents = html.xpath('//ul[@class="videolist"]/li') if (contents.empty?)
+                    
+                    contents.each { |node|
+                        entry = Response.new
+                        
+                        node.search('h3/a').each { |x|
+                            entry.uri = x['href']
+                            entry.title = x['title']
+                        }
+                        
+                        node.css('ul.entry-info li.timestamp').each { |x|
+                            entry.date = Date.strptime(x.content, "%Y/%m/%d")
+                        }
+                        
+                        result.push(entry)
+                    }
+                    
+                    # 無限ループする場合があるので，現在のページから 1件も結果が
+                    # 取得できないときはこれ以上の検索を止める．
+                    @path = nil if (result.empty?)
+                    
+                    return result
+                end
+                
+                # ----------------------------------------------------------- #
+                #
+                #  get_asin_block
+                #
+                #  はてなブックマークの新着エントリーに掲載された Amazon
+                #  商品の各種情報（URL, タイトル，日付）を取得する．
+                #
+                # ----------------------------------------------------------- #
+                def get_asin_block(contents)
+                    result = Array.new
+                    contents.each { |node|
+                        entry = Response.new
+                        
+                        node.xpath('ul[@class="asin_detail"]/li/a').each { |link|
+                            uri = URI.parse(link['href'])
+                            if (uri.host.match(/^(?:www\.)?amazon\.(?:com|ca|co\.uk|de|co\.jp|jp|fr|cn)$/) != nil)
+                                entry.uri = uri.host + '/gp/product/' + asin(uri)
+                                entry.title = link.content
+                                break
+                            end
+                        }
+                        next if (entry.uri == nil)
+                        
+                        node.css('li.asin-info-sub').each { |info|
+                            info.content.scan(/発売日: ([0-9]+\/[0-9]+\/[0-9]+)/) { |date|
+                                entry.date = Date.strptime(date[0], "%Y/%m/%d")
+                            }
+                        }
+                        
+                        result.push(entry)
+                    }
+                    return result
+                end
+                
+                # ----------------------------------------------------------- #
+                #
+                #  asin
+                #
+                #  パスおよびクエリーから ASIN を抽出する．推測方法は，
+                #  /[B0123489][A-Z0-9]{9}/ にマッチする文字列を探すと言う
+                #  方法を採用している．
+                #
+                # ----------------------------------------------------------- #
+                def asin(uri)
+                    if (uri.path != nil)
+                        uri.path.scan(/[B0123489][A-Z0-9]{9}/) { |asin|
+                            return asin if (asin[0].chr == 'B' || check_digit(asin))
+                        }
+                    end
+                    
+                    if (uri.query != nil)
+                        asin = uri.query.match(/[B0123489][A-Z0-9]{9}/)
+                        return asin[0] if (asin != nil)
+                    end
+                    
+                    return nil
+                end
+                
+                # ----------------------------------------------------------- #
+                #
+                #  check_digit
+                #
+                #  ISBN-10 のチェックディジット計算して，有効な ISBN-10 の
+                #  値かどうか判定する．
+                #
+                # ----------------------------------------------------------- #
+                def check_digit(asin)
+                    sum = 0
+                    (0..8).each { |i|
+                        sum += (10 - i) * asin[i].chr.to_i
+                    }
+                    check = 11 - (sum % 11)
+                    check = (check < 10) ? check.to_s : ((check == 10) ? 'X' : '0')
+                    return check == asin[9].chr
+                end
+                
+                # ----------------------------------------------------------- #
+                #  basename
+                # ----------------------------------------------------------- #
+                def basename(options = {})
+                    return '/entrylist' if (options.class != Hash)
+                    
+                    dest = '/'
+                    delimiter = '?'
+                    if (options.has_key?(:type) && @@valid_types.include?(options[:type]))
+                        dest << options[:type].to_s
+                        if (options[:type] == :asin)
+                            dest << '?show_detail=1'
+                            delimiter = '&'
+                        end
+                    else
+                        dest << 'entrylist'
+                    end
+                    
+                    if (options.has_key?(:category) && @@valid_categories.include?(options[:category]))
+                        if (options[:type] == :hotentry && options[:category] == :general)
+                            dest << '?mode=' << options[:category].to_s
+                            delimiter = '&'
+                        else
+                            dest << '/' << options[:category].to_s
+                        end
+                    end
+                    
+                    options.each { |key, value|
+                        case
+                        when (key == :uri)
+                            dest << delimiter << 'url=' << CGI.escape(value)
+                            delimiter = '&'
+                        when (key == :sort && @@valid_sorts.include?(value))
+                            dest << delimiter << key.to_s << '=' << value.to_s
+                            delimiter = '&'
+                        when (key == :threshold)
+                            dest << delimiter << key.to_s << '=' << value.to_s
+                            delimiter = '&'
+                        when (key == :offset)
+                            delimiter = '&'
+                        when (key == :src && @@valid_sources.include?(value))
+                            dest << delimiter << key.to_s << '=' << value.to_s
+                            delimiter = '&'
+                        end
+                    }
+                    
+                    return dest
+                end
+                
+                # ----------------------------------------------------------- #
+                #
+                #  path
+                #
+                #  取得した html の next 属性から次のページへのパスを探す．
+                #
+                # ----------------------------------------------------------- #
+                def path(prev, body)
+                    link = body.scan(/<link rel="next" href="(.+?)".*?>/m).to_s
+                    if (link.empty?)      then return nil;
+                    elsif (link[0] == 47) then return link;
+                    elsif (link[0] == 63) then return prev.gsub(/\?.*$/, "").concat(link);
+                    else return @basename + link;
+                    end
+                end
+            end # URIList
+        end # Bookmark
+    end # Hatena
+end # Crown

data/lib/crown/http-wrapper.rb

@@ -97,7 +97,6 @@ module Crown
                 end
                 return response
             rescue Exception => e
-                puts(e.message)
                 @http.finish if (@http.active?)
                 if (n < @retry_limit)
                     sleep(@retry_interval)