nicoscraper 0.2.4 → 0.2.5

data/lib/classes/searcher.rb

@@ -0,0 +1,248 @@
+# -*- encoding: utf-8 -*-# -*- encoding: utf-8 -*-
+$:.unshift File.dirname(__FILE__)
+
+require 'rubygems'
+require 'ruby-debug'
+require 'time'
+require 'mechanize'
+require 'kconv'
+
+require 'parser.rb'
+
+module Nicos
+  module Searcher
+    class ByTagSuper < Nicos::Connector::Config
+      private
+
+      def get(tag, sort, page, method)
+        paramAry = []
+
+        case sort
+          when 'comment_new'
+            sortStr = ''
+          when 'comment_old'
+            sortStr = 'order=a'
+          when 'view_many'
+            sortStr = 'sort=v'
+          when 'view_few'
+            sortStr = 'sort=v&order=a'
+          when 'comment_many'
+            sortStr = 'sort=r'
+          when 'comment_few'
+            sortStr = 'sort=r&order=a'
+          when 'mylist_many'
+            sortStr = 'sort=m'
+          when 'mylist_few'
+            sortStr = 'sort=m&order=a'
+          when 'post_new'
+            sortStr = 'sort=f'
+          when 'post_old'
+            sortStr = 'sort=f&order=a'
+          when 'length_long'
+            sortStr = 'sort=l'
+          when 'length_short'
+            sortStr = 'sort=l&order=a'
+        end
+
+        paramAry.push("page=#{page}") if page != 1
+        paramAry.push(sortStr)
+        if method == "atom" then paramAry.push("rss=atom&numbers=1") end
+        param = tag + "?" + paramAry.join('&')
+
+        host = 'www.nicovideo.jp'
+        entity = '/tag/' + param
+
+        @connector.get(host, entity)
+      end
+
+      def loop(tag, sort, method, &block)
+        termFlag    = false
+        page        = 1
+        movieObjAry = []
+        order = "continue"
+
+        begin
+           response = get(
+            tag,
+            sort,
+            page,
+            method
+          )
+
+          if response["order"] == "success"
+            result = parse(response["body"])
+            result.each { |each|
+              movie = Nicos::Movie.new(each["video_id"])
+              each["available"] = true
+              movie.set(each)
+              movieObjAry.push(movie)
+            }
+          end
+
+          order = block.call(movieObjAry, page)
+          page += 1
+        end until order != "continue"
+      end
+
+      public
+
+      include Nicos::Connector::SetWait 
+    end
+
+    class ByTagHtml < ByTagSuper
+      def initialize
+        @numOfSearched = 32
+        @incrAmt = 0.2
+
+        @connector = Nicos::Connector.new('mech')
+
+        # HTML中の各パラメータの所在を示すXPath
+        @videoIdXP  = "//div[@class='uad_thumbfrm']/table/tr/td/p/a"
+        @lengthXP   = "//div[@class='uad_thumbfrm']/table/tr/td/p[2]/span"
+        @viewXP     = "//div[@class='uad_thumbfrm']/table/tr/td[2]/div/nobr[1]/strong"
+        @resXP      = "//div[@class='uad_thumbfrm']/table/tr/td[2]/div/nobr[2]/strong"
+        @mylistXP   = "//div[@class='uad_thumbfrm']/table/tr/td[2]/div/nobr[3]/a/strong"
+        @adXP       = "//div[@class='uad_thumbfrm']/table/tr/td[2]/div/nobr[4]/a/strong"
+        @waitConfig = @@waitConfig
+      end
+      attr_accessor :waitConfig
+
+      private
+
+      def parse(movieNum)
+        result = []
+
+        video_id  = /(sm|nm)[0-9]{1,}/.match(@connector.mech.page.search(@videoIdXP)[movieNum]['href'])[0]
+          lengthStr = @connector.mech.page.search(@lengthXP)[movieNum].text.split(/\:/)
+        length    = lengthStr[0].to_i * 60 + lengthStr[1].to_i
+        view      = @connector.mech.page.search(@viewXP)[movieNum]
+                    .text.gsub(/\,/, '').to_i
+        res       = @connector.mech.page.search(@resXP)[movieNum]
+                    .text.gsub(/\,/, '').to_i
+        mylist    = @connector.mech.page.search(@mylistXP)[movieNum]
+                    .text.gsub(/\,/, '').to_i
+        ad        = @connector.mech.page.search(@adXP)[movieNum]
+                    .text.gsub(/\,/, '').to_i
+
+        result.push({
+          "video_id"  => video_id,
+          "length"    => length,
+          "view"      => view,
+          "res"       => res,
+          "mylist"    => mylist,
+          "ad"        => ad
+        })
+      end
+
+      public
+
+      # @param [String] tag
+      # @param [String] sortMethod
+      # @param [HashObj] waitConfig
+      def execute(tag, sortMethod, &block)
+        loop(tag, sort, "mech") { |result, page|
+          block.call(result, page)
+        }
+      end
+    end
+
+    class ByTag < ByTagSuper
+      def initialize
+        @numOfSearched = 32
+        @incrAmt = 0.2
+        @connector = Nicos::Connector::TagAtom.new()
+        @waitConfig = @@waitConfig
+      end
+      attr_accessor :waitConfig
+
+      private 
+
+      def parse(xml)
+        Nicos::Parser.tagAtom(xml)
+      end
+
+      public
+
+      # 実行
+      #
+      # @param [String] tag 検索したいタグ文字列
+      # @param [String] sortMethod ソート方法
+      #==sortMethod: ソート方法
+      # *comment_new*  
+      # コメントが新しい順
+      # 
+      # *comment_old*  
+      # コメントが新しい順
+      # 
+      # *view_many*  
+      # 再生数が多い順
+      # 
+      # *view_few*  
+      # 再生数が少ない順
+      # 
+      # *comment_many*  
+      # コメントが多い順
+      # 
+      # *comment_few*  
+      # コメントが少ない順
+      # 
+      # *mylist_many*  
+      # マイリスト登録が多い順
+      # 
+      # *mylist_few*  
+      # マイリスト登録が少ない順
+      # 
+      # *post_new*  
+      # 登録が新しい順
+      # 
+      # *post_old*  
+      # 登録が少ない順
+      # 
+      # *length_long*  
+      # 再生時間が長い順
+      # 
+      # *length_short*  
+      # 再生時間が短い順
+      # 
+      # @param [HashObj] waitConfig ウェイト設定
+      #==waitConfig: ウェイト設定
+      # <b>ウェイトの変更に際しては、READMEの注意点と免責事項を事前にお読み下さい。</b>
+      #
+      # 以下のフォーマットのハッシュオブジェクトを与えて下さい。これはデフォルト設定です。
+      # また、ハッシュは以下のキーを全て用意する必要はありません。
+      # 変更したい部分のキーと値のみを持つハッシュオブジェクトを作って下さい。
+      #
+      #  @waitConfig = {
+      #    'seqAccLimit' => 10,  # 連続してリクエストする回数
+      #    'afterSeq'    => 10,  # 連続リクエスト後のウェイト（以下、単位は全て秒）
+      #    'each'        => 1,   # 連続リクエスト時の、1リクエスト毎のウェイト
+      #    'increment'   => 1,   # アクセス拒絶時の、次回以降の1リクエスト毎のウェイトの増加量
+      # 
+      #    'deniedSeqReq'=> {     # 連続アクセスを拒否された際の設定（以下同じ）
+      #      'retryLimit'  => 3,  # 再試行の上限回数
+      #      'wait'        => 120 # 次のアクセスまでのウェイト
+      #    },
+      #    
+      #    'serverIsBusy'=> {     # サーバ混雑時
+      #      'retryLimit'  => 3,
+      #      'wait'        => 120
+      #    },
+      #    
+      #    'serviceUnavailable' => { # 503が返ってきた時
+      #      'retryLimit'  => 3,
+      #      'wait'        => 120
+      #    },
+      #    
+      #    'timedOut' => {        # タイムアウト時
+      #      'retryLimit'  => 3,
+      #      'wait'        => 10
+      #    }
+      #  }
+      def execute(tag, sortMethod, &block)
+        loop(tag, sortMethod, "atom") { |result, page|
+          block.call(result, page)
+        }
+      end
+    end
+  end
+end

data/lib/classes/tools.rb

@@ -0,0 +1,15 @@
+require "ruby-debug"
+
+# mixin non destructive
+def mixinND(targetObj, overWriteObj)
+  output = Marshal.load(Marshal.dump(targetObj))
+  if targetObj.instance_of?(Hash)
+    overWriteObj.each_key { |key|    
+      overWriteObj[key] = mixinND(targetObj[key], overWriteObj[key])
+      output[key] = overWriteObj[key]
+    }
+  else
+    output = overWriteObj
+  end
+  return output
+end

data/lib/config/wait.rb

@@ -0,0 +1,63 @@
+module Nicos
+  module Connector
+    class Config
+      @@waitConfigDefault =
+      @@waitConfig = {
+        'seqAccLimit' => 10,  # 連続してリクエストする回数
+        'afterSeq'    => 10,  # 連続リクエスト後のウェイト（以下全て単位は秒）
+        'each'        => 1,   # 連続リクエスト時の、1リクエスト毎のウェイト
+
+        'increment'   => 1,   # アクセス拒絶時の、次回以降の1リクエスト毎のウェイトの増加量
+
+        'deniedSeqReq'=> {    # 連続アクセス拒絶時
+          'retryLimit'  => 3,   # 再試行回数の上限
+          'wait'        => 120  # 再試行までのウェイト
+        },
+        
+        'serverIsBusy'=> {    # サーバ混雑時
+          'retryLimit'  => 3,
+          'wait'        => 120
+        },
+        
+        'serviceUnavailable' => { # 503時
+          'retryLimit'  => 3,
+          'wait'        => 120
+        },
+        
+        'timedOut' => {       # タイムアウト時
+          'retryLimit'  => 3,
+          'wait'        => 10
+        }
+      }
+
+      def Config.setWaitDefault(waitConfig)
+        @@waitConfigDefault = mixinND(
+          @@waitConfigDefault,
+          waitConfig
+        )      
+      end
+
+      def Config.setWait(waitConfig)
+        case waitConfig
+        when "default"
+          @@waitConfig = @@waitConfigDefault
+        when nil
+        else
+          @@waitConfig = mixinND(
+            @@waitConfig,
+            waitConfig
+          )
+        end
+      end
+    end
+
+    module SetWait
+      def setWait(waitConfig)
+        @waitConfig = mixinND(
+          @waitConfig,
+          waitConfig
+        ) if waitConfig != nil
+      end
+    end
+  end
+end

data/lib/nicoscraper.rb

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+module Nicos
+  VERSION = "0.2.5"
+  REPOSITORY =
+    "http://github.com/hdemon/nicoscraper/"
+  AUTHOR = "Masami Yonehara"
+    
+  # nicos.rbが存在する絶対パスを取得
+  ROOT = File.expand_path(File.dirname(__FILE__))
+
+  # 追加で読み込みたいファイルがあればここに記載。
+  # ADDON = File.join(ROOT, '', '')
+
+  #
+  CONFIG_DIR = File.join(ROOT, 'config')
+  CLASSES = File.join(ROOT, 'classes')
+end
+
+# puts Nicos::ROOT
+# puts Nicos::CONFIG_DIR
+
+# Load files.
+[
+  Nicos::CONFIG_DIR, 
+  Nicos::CLASSES
+].each do |path|
+  Dir.glob(File.join(path, '*.rb')).each do |file|
+    require file
+    # puts file
+  end
+end

data/test/movie_spec.rb

@@ -94,7 +94,7 @@ describe "When execute 'Nicos::Searcher::ByTag.execute' method " +
     searcher = Nicos::Searcher::ByTag.new()
     @count = 0
 
-    searcher.execute("ゆっくり実況プレイpart1リンク", "post_old", nil) { |result|
+    searcher.execute("ゆっくり実況プレイpart1リンク", "post_old") { |result|
       @count += 1
       "not continue"
     }
@@ -111,7 +111,7 @@ describe "When execute 'Nicos::Searcher::ByTag.execute' method " +
     searcher = Nicos::Searcher::ByTag.new()
     @count = 0
 
-    searcher.execute("ゆっくり実況プレイpart1リンク", "post_old", nil) { |result|
+    searcher.execute("ゆっくり実況プレイpart1リンク", "post_old") { |result|
       @count += 1
       nil
     }
@@ -128,7 +128,7 @@ describe "When execute 'Nicos::Searcher::ByTag.execute' method " +
     searcher = Nicos::Searcher::ByTag.new()
     count = 0
 
-    searcher.execute("ゆっくり実況プレイpart1リンク", "post_old", nil) { |result|
+    searcher.execute("ゆっくり実況プレイpart1リンク", "post_old") { |result|
       @result = result
 
       count += 1
@@ -161,4 +161,52 @@ describe "When execute 'Nicos::Searcher::ByTag.execute' method " +
     @result[0].comment_num  .should_not be_nil
     @result[0].mylist_counter.should_not be_nil 
   end    
+end
+
+describe "When execute 'Nicos::Connector::setWait" do
+  before(:all) do
+    wait = {
+      'seqAccLimit' => 100,
+     
+      'deniedSeqReq'=> {   
+        'retryLimit'  => 30,  
+        'wait'        => 1200  
+      },
+      
+      'serverIsBusy'=> {   
+        'retryLimit'  => 10
+      }
+    }
+
+    Nicos::Connector::Config::setWait(wait)
+  end
+
+  it "should have following values." do
+    c = Nicos::Searcher::ByTag.new()  
+    c.waitConfig    .should_not be_nil
+    c.waitConfig["seqAccLimit"]
+                    .should == 100
+    c.waitConfig["afterSeq"]
+                    .should == 10
+    c.waitConfig["each"]
+                    .should == 1
+    c.waitConfig["increment"]
+                    .should == 1
+    c.waitConfig["deniedSeqReq"]["retryLimit"]
+                    .should == 30
+    c.waitConfig["deniedSeqReq"]["wait"]
+                    .should == 1200
+    c.waitConfig["serverIsBusy"]["retryLimit"]
+                    .should == 10
+    c.waitConfig["serverIsBusy"]["wait"]
+                    .should == 120
+    c.waitConfig["serviceUnavailable"]["retryLimit"]
+                    .should == 3
+    c.waitConfig["serviceUnavailable"]["wait"]
+                    .should == 120
+    c.waitConfig["timedOut"]["retryLimit"]
+                    .should == 3
+    c.waitConfig["timedOut"]["wait"]
+                    .should == 10
+  end
 end

data/test/searcher_spec.rb

@@ -0,0 +1,207 @@
+# -*- encoding: utf-8 -*-
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'nicoscraper'
+
+describe "When execute 'Nicos::Searcher::ByTag.execute' method " +
+          "and return a string except \"continue\" in this block" do  
+  before(:all) do
+    searcher = Nicos::Searcher::ByTag.new()
+    @count = 0
+
+    searcher.execute("ゆっくり実況プレイpart1リンク", "post_old") { |result|
+      @count += 1
+      "not continue"
+    }
+  end
+
+  it "should end only one access." do
+    @count.should == 1  
+  end
+end
+
+describe "When execute 'Nicos::Searcher::ByTag.execute' method " +
+          "and return a string except \"continue\" in this block" do  
+  before(:all) do
+    searcher = Nicos::Searcher::ByTag.new()
+    @count = 0
+
+    searcher.execute("ゆっくり実況プレイpart1リンク", "post_old") { |result|
+      @count += 1
+      nil
+    }
+  end
+
+  it "should end only one access." do
+    @count.should == 1  
+  end
+end
+
+describe "When execute 'Nicos::Searcher::ByTag.execute' method " +
+          "passing following argument" do  
+  before(:all) do
+    searcher = Nicos::Searcher::ByTag.new()
+    count = 0
+
+    searcher.execute("ゆっくり実況プレイpart1リンク", "post_old") { |result|
+      @result = result
+
+      count += 1
+      puts count
+      "continue" unless count >= 3
+    }
+    puts "end"
+  end
+
+  it "should have Array of movie objects." do
+    @result   .should be_kind_of(Array)
+    @result[0].should be_instance_of(Nicos::Movie)
+  end
+
+  it "should contains movie objects that have following structure." do
+    @result[0].available    .should be_true
+  
+    @result[0].video_id     .should_not be_nil
+    @result[0].title        .should_not be_nil
+    @result[0].create_time  .should_not be_nil
+    @result[0].update_time  .should_not be_nil
+    #@result[0].memo         .should_not be_nil
+    @result[0].description  .should_not be_nil
+    @result[0].thumbnail_url.should_not be_nil
+    @result[0].create_time  .should_not be_nil
+    @result[0].update_time  .should_not be_nil
+    @result[0].length       .should_not be_nil
+
+    @result[0].view_counter .should_not be_nil
+    @result[0].comment_num  .should_not be_nil
+    @result[0].mylist_counter.should_not be_nil 
+  end    
+end
+
+describe "When execute 'Nicos::Connector::setWait" do
+  before(:all) do
+    wait = {
+      'seqAccLimit' => 100,
+     
+      'deniedSeqReq'=> {   
+        'retryLimit'  => 30,  
+        'wait'        => 1200  
+      },
+      
+      'serverIsBusy'=> {   
+        'retryLimit'  => 10
+      }
+    }
+
+    Nicos::Connector::Config::setWait(wait)
+  end
+
+  it "should have following values." do
+    c = Nicos::Searcher::ByTag.new()  
+    c.waitConfig    .should_not be_nil
+    c.waitConfig["seqAccLimit"]
+                    .should == 100
+    c.waitConfig["afterSeq"]
+                    .should == 10
+    c.waitConfig["each"]
+                    .should == 1
+    c.waitConfig["increment"]
+                    .should == 1
+    c.waitConfig["deniedSeqReq"]["retryLimit"]
+                    .should == 30
+    c.waitConfig["deniedSeqReq"]["wait"]
+                    .should == 1200
+    c.waitConfig["serverIsBusy"]["retryLimit"]
+                    .should == 10
+    c.waitConfig["serverIsBusy"]["wait"]
+                    .should == 120
+    c.waitConfig["serviceUnavailable"]["retryLimit"]
+                    .should == 3
+    c.waitConfig["serviceUnavailable"]["wait"]
+                    .should == 120
+    c.waitConfig["timedOut"]["retryLimit"]
+                    .should == 3
+    c.waitConfig["timedOut"]["wait"]
+                    .should == 10
+  end
+
+  after(:all) do
+    Nicos::Connector::Config::setWait("default")
+  end  
+end
+
+describe "When execute 'Nicos::Connector::setWait" do
+  before(:all) do
+    wait = {
+      'seqAccLimit' => 100,
+     
+      'deniedSeqReq'=> {   
+        'retryLimit'  => 30,  
+        'wait'        => 1200  
+      },
+      
+      'serverIsBusy'=> {   
+        'retryLimit'  => 10
+      }
+    }
+
+    @c1 = Nicos::Searcher::ByTag.new()  
+    @c1.setWait(wait)
+
+    @c2 = Nicos::Searcher::ByTag.new()  
+  end
+
+  it "should have following values." do
+    @c1.waitConfig    .should_not be_nil
+    @c1.waitConfig["seqAccLimit"]
+                    .should == 100
+    @c1.waitConfig["afterSeq"]
+                    .should == 10
+    @c1.waitConfig["each"]
+                    .should == 1
+    @c1.waitConfig["increment"]
+                    .should == 1
+    @c1.waitConfig["deniedSeqReq"]["retryLimit"]
+                    .should == 30
+    @c1.waitConfig["deniedSeqReq"]["wait"]
+                    .should == 1200
+    @c1.waitConfig["serverIsBusy"]["retryLimit"]
+                    .should == 10
+    @c1.waitConfig["serverIsBusy"]["wait"]
+                    .should == 120
+    @c1.waitConfig["serviceUnavailable"]["retryLimit"]
+                    .should == 3
+    @c1.waitConfig["serviceUnavailable"]["wait"]
+                    .should == 120
+    @c1.waitConfig["timedOut"]["retryLimit"]
+                    .should == 3
+    @c1.waitConfig["timedOut"]["wait"]
+                    .should == 10
+
+    @c2.waitConfig    .should_not be_nil
+    @c2.waitConfig["seqAccLimit"]
+                    .should == 10
+    @c2.waitConfig["afterSeq"]
+                    .should == 10
+    @c2.waitConfig["each"]
+                    .should == 1
+    @c2.waitConfig["increment"]
+                    .should == 1
+    @c2.waitConfig["deniedSeqReq"]["retryLimit"]
+                    .should == 3
+    @c2.waitConfig["deniedSeqReq"]["wait"]
+                    .should == 120
+    @c2.waitConfig["serverIsBusy"]["retryLimit"]
+                    .should == 3
+    @c2.waitConfig["serverIsBusy"]["wait"]
+                    .should == 120
+    @c2.waitConfig["serviceUnavailable"]["retryLimit"]
+                    .should == 3
+    @c2.waitConfig["serviceUnavailable"]["wait"]
+                    .should == 120
+    @c2.waitConfig["timedOut"]["retryLimit"]
+                    .should == 3
+    @c2.waitConfig["timedOut"]["wait"]
+                    .should == 10
+  end
+end