nicoscraper 0.2.4 → 0.2.5

metadata

@@ -2,7 +2,7 @@
 name: nicoscraper
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors: 
 - Masami Yonehara
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-23 00:00:00 Z
+date: 2011-09-25 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: damerau-levenshtein
@@ -19,13 +19,24 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 0.5.3
   type: :runtime
   prerelease: false
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: rake
+  name: libxml-ruby
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rake
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - "="
@@ -33,10 +44,10 @@ dependencies:
         version: 0.8.7
   type: :development
   prerelease: false
-  version_requirements: *id002
+  version_requirements: *id003
 - !ruby/object:Gem::Dependency 
   name: shoulda
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -44,10 +55,10 @@ dependencies:
         version: "0"
   type: :development
   prerelease: false
-  version_requirements: *id003
+  version_requirements: *id004
 - !ruby/object:Gem::Dependency 
   name: bundler
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -55,10 +66,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *id004
+  version_requirements: *id005
 - !ruby/object:Gem::Dependency 
   name: jeweler
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -66,10 +77,10 @@ dependencies:
         version: 1.6.4
   type: :development
   prerelease: false
-  version_requirements: *id005
+  version_requirements: *id006
 - !ruby/object:Gem::Dependency 
   name: rcov
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -77,7 +88,7 @@ dependencies:
         version: "0"
   type: :development
   prerelease: false
-  version_requirements: *id006
+  version_requirements: *id007
 description: "It scrape movies and mylists of Niconico douga.\n  "
 email: zeitdiebe@gmail.com
 executables: []
@@ -96,15 +107,18 @@ files:
 - Rakefile
 - VERSION
 - index.html
-- lib/connector.rb
-- lib/converter.rb
-- lib/movie.rb
-- lib/mylist.rb
-- lib/namespace.rb
-- lib/parser.rb
-- lib/searcher.rb
-- nicoscraper.gemspec
+- lib/classes/connector.rb
+- lib/classes/converter.rb
+- lib/classes/header.rb
+- lib/classes/movie.rb
+- lib/classes/mylist.rb
+- lib/classes/parser.rb
+- lib/classes/searcher.rb
+- lib/classes/tools.rb
+- lib/config/wait.rb
+- lib/nicoscraper.rb
 - test/movie_spec.rb
+- test/searcher_spec.rb
 homepage: http://github.com/hdemon/nicoscraper
 licenses: 
 - MIT
@@ -118,7 +132,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 893829155072320695
+      hash: 2919755010107136156
       segments: 
       - 0
       version: "0"

data/lib/connector.rb

@@ -1,364 +0,0 @@
-# -*- encoding: utf-8 -*-
-$:.unshift File.dirname(__FILE__) 
-
-require 'rubygems'
-require 'ruby-debug'
-require 'net/http'
-
-require 'namespace.rb'
-
-module Nicos::Connector
-  class Connector
-    def initialize
-      # デフォルトのウェイト設定
-      @seqTime = 0
-
-      @waitConfig = {
-        'seqAccLimit' => 10,  # 連続してリクエストする回数
-        'afterSeq'    => 10,  # 連続リクエスト後のウェイト
-        'each'        => 1,   # 連続リクエスト時の、1リクエスト毎のウェイト
-
-        'increment'   => 1,   # アクセス拒絶時の、次回以降の1リクエスト毎のウェイトの増加量
-        ''            => 100,
-
-        'deniedSeqReq'=> {
-          'retryLimit'  => 3,
-          'wait'        => 120
-        },
-        
-        'serverIsBusy'=> {
-          'retryLimit'  => 3,
-          'wait'        => 120
-        },
-        
-        'serviceUnavailable' => {
-          'retryLimit'  => 3,
-          'wait'        => 120
-        },
-        
-        'timedOut' => {
-          'retryLimit'  => 3,
-          'wait'        => 10
-        }
-      }
-
-      @result = {}
-    end
-
-    private
-    
-    def notPublic
-      # マイリスト非公開のときに403になる。後で専用の処理を入れるべき。
-      puts "This movie/mylist is not public."
-      @result = "notPublic"
-      return { "order" => "terminate" }  
-    end
-
-    def limInCommunity
-      puts "This movie/mylist is limited in comunity members."
-      # ex. item_id -> 1294702905
-      @result = "limInCommunity"
-      return { "order" => "terminate" }   
-    end
-
-    def notFound
-      puts "This movie/mylist is not found."
-      @result = "notFound"
-      return { "order" => "terminate" }  
-    end
-
-    def deleted
-      puts "This movie/mylist is deleted."
-      @result = "deleted"
-      return { "order" => "terminate" }  
-    end
-
-    def deniedSeqReq
-      puts "Denied sequential requests."
-      sleep @waitConfig["deniedSeqReq"]
-      @result = "deniedSeqReq"
-      return { "order" => "retry" }  
-    end
-
-    def serverIsBusy
-      puts "The server is busy."
-      sleep @waitConfig["serverIsBusy"]
-      @result = "serverIsBusy"
-      return { "order" => "retry" } 
-    end
-
-    def serviceUnavailable
-      puts "Service unavailable."
-      sleep @waitConfig["serviceUnavailable"]
-      @result = "serviceUnavailable"
-      return { "order" => "retry" } 
-    end
-
-    def timedOut
-      puts "Request timed out."
-      sleep @waitConfig["timedOut"]
-      @result = "timedOut"
-      return { "order" => "retry" } 
-    end
-
-    def success(resBody)
-      sleep @waitConfig["each"]
-      @seqTime += 1
-      
-      if @seqTime >= @waitConfig["seqAccLimit"]
-        sleep @waitConfig["afterSeq"]
-        @seqTime = 0
-      end
-      return { "order" => "success", "body" => resBody } 
-    end
-
-    def wait(status)
-      puts "Wait for " + waitTime + " second."
-      sleep @waitConfig[status.to_s]
-    end
-      
-    public
-
-    def setWait(waitConfig)
-      if waitConfig != nil
-        @waitConfig = mixin(@waitConfig, waitConfig)
-      end
-    end
-  end
-
-  class Xml < Connector
-    def get (host, entity)  
-      response = nil
-            
-      begin
-        puts "Request to " + host + entity 
-        Net::HTTP.start(host, 80) { |http|
-          response = http.get(entity)
-        }
-
-      rescue => e
-        puts e
-      rescue Timeout::Error => e  
-        timeOut      
-
-      else
-        res = case response
-        when Net::HTTPSuccess
-          reviewRes( response.body.force_encoding("UTF-8") )
-        #    return response.body.force_encoding("UTF-8") 
-        # when Net::HTTPRedirection
-        #  fetch(response['location'], limit - 1)
-        when Net::HTTPForbidden
-          forbidden            
-        when Net::HTTPNotFound
-          notFound
-        when Net::HTTPServiceUnavailable 
-          serviceUnavailable
-        else
-          unknownError
-        end    
-      end until res["order"] == "success" ||
-                res["order"] == "terminate"
-
-      res
-    end
-  end
-
-  class MylistAtom < Xml
-    private
-
-    def forbidden
-      # マイリストが非公開の場合、html/Atomのどちらへのリクエストであっても、403が返ってくる。
-      notPublic
-    end
-
-    def reviewRes(resBody)
-      if # アクセス集中時
-        /大変ご迷惑をおかけいたしますが、しばらく時間をあけてから再度検索いただくようご協力をお願いいたします。/ =~
-          resBody.force_encoding("UTF-8")
-      then
-        serverIsBusy
-      else
-        success(resBody)
-      end      
-    end
-  end
-
-  class TagAtom < Xml
-    private
-
-    def forbidden
-      # マイリストが非公開の場合、html/Atomのどちらへのリクエストであっても、403が返ってくる。
-      notPublic
-    end
-
-    def reviewRes(resBody)
-      if # アクセス集中時
-        /大変ご迷惑をおかけいたしますが、しばらく時間をあけてから再度検索いただくようご協力をお願いいたします。/ =~
-          resBody.force_encoding("UTF-8")
-      then
-        serverIsBusy
-      else
-        success(resBody)
-      end      
-    end
-  end
-
-  class GetThumbInfo < Xml
-    private
-
-    def reviewRes(resBody)
-      r = resBody.force_encoding("UTF-8")
-
-      if # getThumbInfoは、該当する動画がない・削除済み・コミュニティ限定でも200が返ってくる。
-        /<nicovideo_thumb_response\sstatus=\"fail\">/ =~ r
-        if /<code>NOT_FOUND<\/code>/ =~ r
-          notFound
-        elsif /<code>DELETED<\/code>/ =~ r
-          deleted
-        elsif /<code>COMMUNITY<\/code>/ =~ r
-          limInCommunity
-        else
-          serverIsBusy
-        end
-      else
-        success(resBody)
-      end      
-    end
-  end
-
-=begin
-  class HtmlConnector < Connector
-    def initialize(mode)
-      @mode = mode
-      # デフォルトのウェイト設定
-      @waitConfig = {
-        'consec_count'  => 10,  # 連続してリクエストする回数
-        'consec_wait'   => 10,  # 連続リクエスト後のウェイト
-        'each'          => 10,  # 連続リクエスト時の、1リクエスト毎のウェイト
-   
-        '200-abnormal'  => 300, # アクセス拒絶時（「短時間での連続アクセスは・・・」）の場合の再試行までの時間
-        'unavailable'   => 10,
-        '403'           => 300, # "403"時の再試行までのウェイト
-        '404'           => 300, # "403"時の再試行までのウェイト
-        'increment'     => 1,   # アクセス拒絶時の、次回以降の1リクエスト毎のウェイトの増加量
-
-        'timeout'       => 10,  # タイムアウト時の、再試行までのウェイト
-        '500'           => 10,  # "500"時の再試行までのウェイト
-        '503'           => 10,  # "503"時の再試行までのウェイト
-   
-        'retryLimit'    => 3    # 再試行回数の限度
-      }
-      
-      # 1つの検索結果画面に表示される動画の数。現時点では32個がデフォルトの模様。
-      @NumOfSearched = 32
-
-      @mech = Mechanize.new
-      # メモリ節約のため、Mechanizeの履歴機能を切る。
-      @mech.max_history = 1
-
-      @consec_count = 0
-    end
-    
-    public 
-
-    def errorStatus(ex)
-      # 再試行回数が
-      @retryTime += 1
-      if @retryTime >= @wait['allowance_time']
-        return false
-      end
-
-      case ex.response_code
-      when '403' then
-        sleep @wait['403']
-        warn "403"
-      when '500' then
-        sleep @wait['500']
-        warn "500"
-      when '503' then
-        sleep @wait['503']
-        warn "503"
-      else
-        warn "Server error: #{ex.code}"
-        return false
-      end
-   
-      @connection = false
-      @failed += 1
-    end
-
-    def htmlReq (url, request, procedure)
-      @failed = 0
-
-      # 再試行ループ
-      begin
-        eachWait 
-        @connection = nil
-        request.call(url)
-      
-      # タイムアウト時処理
-      rescue TimeoutError
-        timeOut
-        retry
-         
-      # Mechanizeでアクセスし、200以外のステータスが返ってきた時
-      # 実際に該当するコードが返ってきたことがないので、正常に動くか不明   
-      rescue Mechanize::ResponseCodeError => ex
-        if errorStatus(ex) then retry
-        else break end
-         
-      # HTTP Status:200時の処理
-      else
-        procedure.call
-        
-        # 失敗カウントが指定回数を超えたらループを終わる。
-        if @failed >= @wait['allowance_time'] then
-          puts 'Exceeded the limit of retry time.'
-          @connection = false
-          break
-        end
-      end until @connection
-
-      # 連続アクセスカウント+1
-      @consec_count += 1
-      # 成功 = true / 失敗 = false
-      return @connection
-    end
-
-    def htmlGet (host, entity)
-      htmlReq(
-        host + entity,
-        lambda { |url|
-          t = Thread.new do
-            @mech.get(url)
-            puts "Requesting for " + url
-          end
-          t.join
-        },
-        # HTTP Status:200時の処理 
-        lambda {
-          # 連続アクセス拒絶メッセージが返ってきた時       
-          if /短時間での連続アクセスはご遠慮ください/ =~ @mech.page.search('/html').text then
-            puts 'Access rejected.'
-            @connection = false
-            @failed += 1
-       
-            # ウェイトを置いた後、今後のページ毎のウェイトを増やす。
-            puts 'Waiting for ' + @wait['rejected'] + 's.'
-            sleep @wait['rejected']
-            @wait['each'] += @wait['increment']
-            puts 'Increased each @wait by ' + @wait['increment'] + 'sec.'
-          else
-            @connection = true
-          end
-        }
-      )
-      
-      return @mech.page
-    end
-    
-    attr_reader :mech
-  end
-=end
-end

data/lib/converter.rb

@@ -1,72 +0,0 @@
-# -*- encoding: utf-8 -*-
-$:.unshift File.dirname(__FILE__) 
-
-require 'rubygems'
-require 'xml'
-require 'time'
-require 'namespace.rb'
-
-module Nicos::Converter #:nodoc:
-  def iso8601ToUnix(str)
-    Time.strptime(str, "%Y-%m-%dT%H:%M:%S").to_i
-  end
-  module_function :iso8601ToUnix
-
-  def japToUnix(str)
-    str.gsub!(/年|月/, '-')
-      .gsub!(/日/, 'T')
-      .gsub!(/：/, ':')
-      .gsub!(/\s/, '')
-    iso8601ToUnix(str)
-  end
-  module_function :japToUnix
-    
-  def toSeconds(lengthStr)
-    # lengthStr = "mm:ss"
-    lengthStr = lengthStr.split(/\:/)
-    lengthStr[0].to_i * 60 + lengthStr[1].to_i
-  end
-  module_function :toSeconds
-  
-  def commaRemover(str)
-    str.gsub(/\,/, '').to_i
-  end  
-  module_function :commaRemover
-end
-
-module Nicos::Extractor #:nodoc:
-  def mylistId(str)
-    /(mylist\/)([0-9]{1,})/ =~ str
-    $2.to_i
-  end
-  module_function :mylistId
-
-  def itemId(str)
-    /(watch\/)([0-9]{1,})/ =~ str
-    $2.to_i
-  end
-  module_function :itemId
-  
-  def videoId(str)
-    /(http:\/\/www.nicovideo.jp\/watch\/)((sm|nm)[0-9]{1,})/ =~ str
-    $2    
-  end
-  module_function :videoId
-end
-
-module Nicos::Unicode
-  def escape(str)
-    ary = str.unpack("U*").map!{|i| "\\u#{i.to_s(16)}"}
-    ary.join
-  end
-  
-  UNESCAPE_WORKER_ARRAY = []
-  def unescape(str)
-    str.gsub(/\\u([0-9a-f]{4})/) {
-      UNESCAPE_WORKER_ARRAY[0] = $1.hex
-      UNESCAPE_WORKER_ARRAY.pack("U")
-    }
-  end
-  
-  module_function :escape, :unescape
-end