nicoscraper 0.1.0 → 0.2.0

data/Rakefile

@@ -51,4 +51,5 @@ Rake::RDocTask.new do |rdoc|
   rdoc.title = "nicoscraper #{version}"
   rdoc.rdoc_files.include('README*')
   rdoc.rdoc_files.include('lib/**/*.rb')
+  rdoc.rdoc_files.include('lib/*.rb')
 end

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/lib/connector.rb

@@ -1,86 +1,264 @@
 # -*- encoding: utf-8 -*-
+$:.unshift File.dirname(__FILE__) 
+
 require 'rubygems'
 require 'ruby-debug'
 require 'net/http'
 
+
 class Connector
-  def initialize(mode)
-    @mode = mode
+  def initialize
     # デフォルトのウェイト設定
+    @seqTime = 0
+
     @waitConfig = {
-      'consec_count'  => 10,    # 連続してリクエストする回数
-      'consec_wait'   => 10,    # 連続リクエスト後のウェイト
-      'each'          => 10,    # 連続リクエスト時の、1リクエスト毎のウェイト
- 
-      '200-abnormal'  => 1,   # アクセス拒絶時（「短時間での連続アクセスは・・・」）の場合の再試行までの時間
-      'unavailable'   => 10,
-      '403'           => 1,   # "403"時の再試行までのウェイト
-      '404'           => 1,   # "403"時の再試行までのウェイト
-      'increment'     => 1,     # アクセス拒絶時の、次回以降の1リクエスト毎のウェイトの増加量
+      'seqAccLimit' => 10,  # 連続してリクエストする回数
+      'afterSeq'    => 10,  # 連続リクエスト後のウェイト
+      'each'        => 1,   # 連続リクエスト時の、1リクエスト毎のウェイト
 
-      'timeout'       => 5,     # タイムアウト時の、再試行までのウェイト
-      '500'           => 1,   # "500"時の再試行までのウェイト
-      '503'           => 1,   # "503"時の再試行までのウェイト
- 
-      'retryLimit'    => 5    # 再試行回数の限度
+      'increment'   => 1,   # アクセス拒絶時の、次回以降の1リクエスト毎のウェイトの増加量
+      ''            => 100,
+
+      'deniedSeqReq'=> {
+        'retryLimit'  => 3,
+        'wait'        => 120
+      },
+      
+      'serverIsBusy'=> {
+        'retryLimit'  => 3,
+        'wait'        => 120
+      },
+      
+      'serviceUnavailable' => {
+        'retryLimit'  => 3,
+        'wait'        => 120
+      },
+      
+      'timedOut' => {
+        'retryLimit'  => 3,
+        'wait'        => 10
+      }
     }
-    
-    # 1つの検索結果画面に表示される動画の数。現時点では10個。
-    @NumOfSearched = 32
-    
-    if @mode == "mech"
-      @mech = Mechanize.new
-      # メモリ節約のため、Mechanizeの履歴機能をオフにする。
-      @mech.max_history = 1
-    end
 
-    @consec_count = 0
+    @result = {}
   end
 
   private
   
-  def mixin(targetObj, overWriteObj)
-    output = Marshal.load(Marshal.dump(targetObj))
-    if targetObj.instance_of?(Hash)
-      overWriteObj.each_key { |key|    
-        overWriteObj[key] = mixin(targetObj[key], overWriteObj[key])
-        output[key] = overWriteObj[key]
-      }
-    else
-      output = overWriteObj
+  def notPublic
+    # マイリスト非公開のときに403になる。後で専用の処理を入れるべき。
+    puts "This movie/mylist is not public."
+    @result = "notPublic"
+    return { "order" => "terminate" }  
+  end
+
+  def limInCommunity
+    puts "This movie/mylist is limited in comunity members."
+    # ex. item_id -> 1294702905
+    @result = "limInCommunity"
+    return { "order" => "terminate" }   
+  end
+
+  def notFound
+    puts "This movie/mylist is not found."
+    @result = "notFound"
+    return { "order" => "terminate" }  
+  end
+
+  def deleted
+    puts "This movie/mylist is deleted."
+    @result = "deleted"
+    return { "order" => "terminate" }  
+  end
+
+  def deniedSeqReq
+    puts "Denied sequential requests."
+    sleep @waitConfig["deniedSeqReq"]
+    @result = "deniedSeqReq"
+    return { "order" => "retry" }  
+  end
+
+  def serverIsBusy
+    puts "The server is busy."
+    sleep @waitConfig["serverIsBusy"]
+    @result = "serverIsBusy"
+    return { "order" => "retry" } 
+  end
+
+  def serviceUnavailable
+    puts "Service unavailable."
+    sleep @waitConfig["serviceUnavailable"]
+    @result = "serviceUnavailable"
+    return { "order" => "retry" } 
+  end
+
+  def timedOut
+    puts "Request timed out."
+    sleep @waitConfig["timedOut"]
+    @result = "timedOut"
+    return { "order" => "retry" } 
+  end
+
+  def success(resBody)
+    sleep @waitConfig["each"]
+    @seqTime += 1
+    
+    if @seqTime >= @waitConfig["seqAccLimit"]
+      sleep @waitConfig["afterSeq"]
+      @seqTime = 0
     end
-    return output
+    return { "order" => "success", "body" => resBody } 
   end
-  
-  public 
-  
+
+  def wait(status)
+    puts "Wait for " + waitTime + " second."
+    sleep @waitConfig[status.to_s]
+  end
+    
+  public
+
   def setWait(waitConfig)
     if waitConfig != nil
       @waitConfig = mixin(@waitConfig, waitConfig)
     end
   end
+end
 
-  def eachWait
-     # ウェイト...1回目の場合は無視 -------------------------
-    if @consec_count != 0
-      # 動画毎
-      sleep @wait['each']
- 
-      # 一定のリクエスト回数毎
-      if @consec_count >= @wait['consec_count'] then
-        sleep @wait['consec_wait']
-        @consec_count = 0
+
+class XmlConnector < Connector
+  def get (host, entity)  
+    response = nil
+          
+    begin
+      puts "Request to " + host + entity 
+      Net::HTTP.start(host, 80) { |http|
+        response = http.get(entity)
+      }
+
+    rescue => e
+      puts e
+    rescue Timeout::Error => e  
+      timeOut      
+
+    else
+      res = case response
+      when Net::HTTPSuccess
+        reviewRes( response.body.force_encoding("UTF-8") )
+      #    return response.body.force_encoding("UTF-8") 
+      # when Net::HTTPRedirection
+      #  fetch(response['location'], limit - 1)
+      when Net::HTTPForbidden
+        forbidden            
+      when Net::HTTPNotFound
+        notFound
+      when Net::HTTPServiceUnavailable 
+        serviceUnavailable
+      else
+        unknownError
+      end    
+    end until res["order"] == "success" ||
+              res["order"] == "terminate"
+
+    res
+  end
+end
+
+class MylistAtomConnector < XmlConnector
+  private
+
+  def forbidden
+    # マイリストが非公開の場合、html/Atomのどちらへのリクエストであっても、403が返ってくる。
+    notPublic
+  end
+
+  def reviewRes(resBody)
+    if # アクセス集中時
+      /大変ご迷惑をおかけいたしますが、しばらく時間をあけてから再度検索いただくようご協力をお願いいたします。/ =~
+        resBody.force_encoding("UTF-8")
+    then
+      serverIsBusy
+    else
+      success(resBody)
+    end      
+  end
+end
+
+class SearchByTagAtomConnector < XmlConnector
+  private
+
+  def forbidden
+    # マイリストが非公開の場合、html/Atomのどちらへのリクエストであっても、403が返ってくる。
+    notPublic
+  end
+
+  def reviewRes(resBody)
+    if # アクセス集中時
+      /大変ご迷惑をおかけいたしますが、しばらく時間をあけてから再度検索いただくようご協力をお願いいたします。/ =~
+        resBody.force_encoding("UTF-8")
+    then
+      serverIsBusy
+    else
+      success(resBody)
+    end      
+  end
+end
+
+class GetThumbInfoConnector < XmlConnector
+  private
+
+  def reviewRes(resBody)
+    r = resBody.force_encoding("UTF-8")
+
+    if # getThumbInfoは、該当する動画がない・削除済み・コミュニティ限定でも200が返ってくる。
+      /<nicovideo_thumb_response\sstatus=\"fail\">/ =~ r
+      if /<code>NOT_FOUND<\/code>/ =~ r
+        notFound
+      elsif /<code>DELETED<\/code>/ =~ r
+        deleted
+      elsif /<code>COMMUNITY<\/code>/ =~ r
+        limInCommunity
+      else
+        serverIsBusy
       end
-    end
-    # ------------------------------------------------
+    else
+      success(resBody)
+    end      
   end
+end
 
-  def timeOut
-    sleep @wait['timeout']
-    @connection = false
-    @failed += 1
-    warn "Timeout"
+class HtmlConnector < Connector
+  def initialize(mode)
+    @mode = mode
+    # デフォルトのウェイト設定
+    @waitConfig = {
+      'consec_count'  => 10,  # 連続してリクエストする回数
+      'consec_wait'   => 10,  # 連続リクエスト後のウェイト
+      'each'          => 10,  # 連続リクエスト時の、1リクエスト毎のウェイト
+ 
+      '200-abnormal'  => 300, # アクセス拒絶時（「短時間での連続アクセスは・・・」）の場合の再試行までの時間
+      'unavailable'   => 10,
+      '403'           => 300, # "403"時の再試行までのウェイト
+      '404'           => 300, # "403"時の再試行までのウェイト
+      'increment'     => 1,   # アクセス拒絶時の、次回以降の1リクエスト毎のウェイトの増加量
+
+      'timeout'       => 10,  # タイムアウト時の、再試行までのウェイト
+      '500'           => 10,  # "500"時の再試行までのウェイト
+      '503'           => 10,  # "503"時の再試行までのウェイト
+ 
+      'retryLimit'    => 3    # 再試行回数の限度
+    }
+    
+    # 1つの検索結果画面に表示される動画の数。現時点では32個がデフォルトの模様。
+    @NumOfSearched = 32
+
+    @mech = Mechanize.new
+    # メモリ節約のため、Mechanizeの履歴機能を切る。
+    @mech.max_history = 1
+
+    @consec_count = 0
   end
+  
+  public 
 
   def errorStatus(ex)
     # 再試行回数が
@@ -178,92 +356,5 @@ class Connector
     return @mech.page
   end
 
-  def xmlGet (host, entity)
-    response = nil
-    xmlDoc = nil
-    retryCount = 0
-    terminate = false
-        
-    begin
-      puts "Requesting to " + host + entity 
-      Net::HTTP.start(host, 80) { |http|
-        response = http.get(entity)
-      }
-    rescue => e
-      puts e
-    rescue Timeout::Error => e  
-      puts e
-      puts "Timeout."
-      # マイリスト非公開のときに、403になる。後で専用の処理を入れるべき。
-      wait("timeout")
-      retryCount += 1
-      
-      if retryCount >= @waitConfig["retryLimit"]
-        terminate = true
-        return "failed"
-      end
-    else
-      case response
-      when Net::HTTPSuccess
-        unless abnormalRes(response.body)
-          terminate = true
-          return response.body.force_encoding("UTF-8") 
-        end
-        wait("200-abnormal")
-        retryCount += 1
-      when Net::HTTPRedirection 
-        fetch(response['location'], limit - 1)
-      when Net::HTTPForbidden
-        puts "Access forbidden."
-        # マイリスト非公開のときに、403になる。後で専用の処理を入れるべき。
-        wait("403")
-        retryCount += 1
-      when Net::HTTPNotFound
-        puts "Http not found."
-        wait("404")
-        retryCount += 1
-      when Net::HTTPServiceUnavailable  
-        puts "Access rejected or service unavailable."
-        wait("unavailable")
-        retryCount += 1
-      else
-        puts response.force_encoding("UTF-8") 
-        puts "Unknown error."
-        wait("other")
-        retryCount += 1
-      end
-      
-      if retryCount >= @waitConfig["retryLimit"]
-        terminate = true
-        return "failed"
-      end
-    end until terminate
-  end
-
-  def abnormalRes(resBody)
-    if
-      # mylistRss アクセス集中時
-       /大変ご迷惑をおかけいたしますが、しばらく時間をあけてから再度検索いただくようご協力をお願いいたします。/ =~ resBody.force_encoding("UTF-8") ||
-      # getThumbInfo失敗時
-       /<nicovideo_thumb_response\sstatus=\"fail\">/ =~ resBody
-    then
-      puts "!!!!"
-      true
-    end  
-  end
-  
-  def wait(status)
-    sleep @waitConfig[status.to_s]
-  end
-  
-  def get (host, entity)
-    case @mode
-    when "html"
-      mechGet(host + entity)
-    when "atom"
-      xmlGet(host, entity)
-    end
-  end
-
   attr_reader :mech
 end

data/lib/movie.rb

@@ -1,12 +1,14 @@
 # -*- encoding: utf-8 -*-
+$:.unshift File.dirname(__FILE__) 
+
 require 'rubygems'
 require 'ruby-debug'
 require 'damerau-levenshtein'
 require 'kconv'
 
-require 'parser'
-require 'mylist'
-require 'connector'
+require 'parser.rb'
+require 'mylist.rb'
+require 'connector.rb'
   
 class Movie  
   def initialize(video_id)
@@ -19,9 +21,9 @@ class Movie
   public
   
   # 指定されたマイリストに自分が入っていれば、真を返す。  
-  def isBelongsTo (_mylistId, &block)
+  def isBelongsTo (mylistId, &block)
     isBelongs = false
-    thisMl = Mylist.new(_mylistId)
+    thisMl = Mylist.new(mylistId)
     thisMl.getInfoLt
     
     thisMl.movies.each { |movie|
@@ -29,14 +31,14 @@ class Movie
     }   
   
     if isBelongs
-      puts "\sThis movie is found in mylist/" + _mylistId
+      puts "\sThis movie is found in mylist/" + mylistId.to_s
     else
-      puts "\sThis movie is not found in mylist/" + _mylistId
+      puts "\sThis movie is not found in mylist/" + mylistId.to_s
     end
     
     # 無駄なアクセスを省くため、マイリスト中の動画に関する追加処理があれば、
     # ブロックとして実行できる。
-    block.call(thisMl)
+    block.call(thisMl) if block != nil
     
     return isBelongs
   end
@@ -63,7 +65,7 @@ class Movie
     
     mylistIdAry.each { |_mylistId|
       belongsTo = isBelongsTo(_mylistId) { |mylistObj|        
-        similarity = mylistObj.isSeries 
+        similarity = mylistObj.getSimilarity 
         puts "\sSimilarity:\t" + similarity.to_s
       }
       puts belongsTo
@@ -103,22 +105,20 @@ class Movie
   end
   
   def getInfo
-    con = Connector.new('xml')
+    con = GetThumbInfoConnector.new()
     host = 'ext.nicovideo.jp'
     entity = '/api/getthumbinfo/' + @video_id
     con.setWait(nil)
-    xml = con.xmlGet(host, entity)
-    
-    unless
-      xml =~ /<nicovideo_thumb_response\sstatus=\"fail\">/ ||
-      xml == "failed"
+    result = con.get(host, entity)
+
+    if
+      result["order"] == "success"
     then
-      param = NicoParser.getThumbInfo(xml)
-      set(param)
+      parsed = NicoParser.getThumbInfo(result["body"])
+      set(parsed)
       @available = true
     else
       @available = false
-      return "failed"
     end
   end
     
@@ -239,8 +239,8 @@ class Movie
         @thumb_type = param
     	when "embeddable"
         @embeddable = param
-    	when "movieNum_live_play"
-        @movieNum_live_play = param
+    	when "no_live_play"
+        @no_live_play = param
     	when "tags_jp"
         @tags_jp = param
       when "tags_tw"
@@ -290,7 +290,7 @@ class Movie
   attr_accessor	:watch_url
   attr_accessor	:thumb_type
   attr_accessor	:embeddable
-  attr_accessor	:movieNum_live_play
+  attr_accessor	:no_live_play
   attr_accessor	:tags_jp
   attr_accessor	:tags_tw
   attr_accessor	:user_id

data/lib/mylist.rb

@@ -1,11 +1,13 @@
 # -*- encoding: utf-8 -*-
+$:.unshift File.dirname(__FILE__) 
+
 require 'rubygems'
 require 'ruby-debug'
 require 'kconv'
 
-require 'parser'
-require 'movie'
-require 'connector'
+require 'parser.rb'
+require 'movie.rb'
+require 'connector.rb'
 
 
 class Mylist
@@ -15,49 +17,9 @@ class Mylist
     @available  = false
   end
   
-  def simOfTitle  
-    match = false
-    dl = DamerauLevenshtein
-    d = 0.0
-    
-    puts "matching..."
-    
-    # O(n^2)なのでどうにかしたい。しかし、最大でも500C2=125000なので、
-    # 日々の利用については許容できると思う、
-    @movies.each { |myself|
-      @movies.each { |amovieNumther|
-        _d = dl.distance(myself.title, amovieNumther.title)
-        d += _d
-      }
-    }
-    
-    similarity = 1 - ( (d / @movies.length) / title.length )
-    puts "Similarity: " + similarity.to_s
-    similarity
-  end 
-
-  def userId
-    idGroup = {}
-    samePublisher = false
-    threshold = 0.9
-    
-    @movies.each { |movie|
-      idGroup[movie.user_id] += 1
-    }
-    
-    idGroup.each { |group|
-      if @movies.length / threshold < group.length
-        samePublicher = true  
-      end
-    }
-    
-    return samePublisher    
-  end
-  
-  # 自分がシリーズをまとめたマイリストであるかを判定する。
-  # 判定基準は、1.一定数以上の動画の投稿者が、マイリスト作成者と同じであること。
-  # 2.タイトルの類似度が、定められた基準以上であること。
-  def isSeries
+  # 自分に含まれている動画のタイトルをすべての組み合わせにおいて比較し、
+  # 類似度の平均を返す。
+  def getSimilarity
     l = @movies.length - 1
     dlc = DamerauLevenshtein
     dl = 0.0
@@ -180,17 +142,17 @@ class Mylist
   end
   
   def getInfoLt
-    con = Connector.new('xml')
+    con = MylistAtomConnector.new()
     host = 'www.nicovideo.jp'
     puts @mylist_id
     entity = '/mylist/' + @mylist_id.to_s + '?rss=atom&numbers=1'
     con.setWait(nil)
-    xml = con.xmlGet(host, entity)
-    
-    unless
-      xml == "failed"
-    then  
-      parsed = NicoParser.mylistRss(xml)
+    result = con.get(host, entity)
+
+    if
+      result["order"] == "success"
+    then
+      parsed = NicoParser.mylistRss(result["body"])
       
       parsed["entry"].each { |e|
         movie = Movie.new(e["video_id"])
@@ -201,7 +163,9 @@ class Mylist
       
       set(parsed["mylist"])
       @available = true
-    end    
+    else
+      @available = false
+    end  
   end  
 
   def set(paramObj)
@@ -252,7 +216,8 @@ class Mylist
 	attr_accessor :create_time  
 	attr_accessor :update_time  
 	attr_accessor :icon_id    
-	attr_accessor :sort_order   
+  attr_accessor :sort_order   
+  attr_accessor :author   
   
   attr_accessor :movies     
 end

data/lib/nicoscraper.rb

@@ -3,4 +3,4 @@ require 'ruby-debug'
 
 require 'movie'
 require 'mylist'
-require 'getmovie'
+require 'searcher'

data/lib/parser.rb

@@ -1,8 +1,10 @@
 # -*- encoding: utf-8 -*-
+$:.unshift File.dirname(__FILE__) 
+
 require 'rubygems'
 require 'xml'
 require 'time'
-require 'converter'
+require 'converter.rb'
 
 module NicoParser
   public
@@ -152,7 +154,7 @@ module NicoParser
           end
         when "subtitle"  
           doc.read
-          parsed["entry"][n]["description"] = doc.value
+          parsed["mylist"]["description"] = doc.value
         when "id"
           if n == -1
             doc.read
@@ -161,7 +163,7 @@ module NicoParser
           else
             doc.read
             parsed["entry"][n]["item_id"] =
-                Extract.itemId(doc.value)
+              Extract.itemId(doc.value)
           end    
         when "updated"
           doc.read
@@ -176,54 +178,30 @@ module NicoParser
         when "content"
           doc.read
           html = doc.value   
-      
-          memo =
-            html.slice(
-              /<p\sclass\=\"nico-memo\"\>[^\<]{1,}/
-            ).to_s.slice(21, 999)
-          
+
+          /(<p\sclass=\"nico-memo\"\>)([^\<]{1,})/ =~ html
+          memo = $2
+                   
           /(<p\sclass=\"nico-thumbnail\">.+src=\")(http:\/\/[^\"]{1,})/ =~ html
           thumbnail_url = $2
-                                     
-          description = 
-            html.slice(
-              /<p\sclass\=\"nico-description\"\>[^\<]{1,}/
-            ).to_s.slice(31, 999)  
-            
-          length = 
-            Convert.toSeconds(
-              html.slice(
-                /<strong\sclass\=\"nico-info-length\"\>[^\<]{1,}/
-              ).to_s.slice(33, 999)
-            )       
-             
-          first_retrieve =
-            Convert.japToUnix(
-              html.slice(
-                /<strong\sclass\=\"nico-info-date\"\>[^\<]{1,}/
-              ).to_s.slice(31, 999)
-            )    
-            
-          view = 
-            Convert.commaRemover(
-              html.slice(
-                /<strong\sclass\=\"nico-numbers-view\"\>[^\<]{1,}/
-              ).to_s.slice(34, 999)
-            ) 
-                        
-          res =
-            Convert.commaRemover(
-              html.slice(
-                /<strong\sclass\=\"nico-numbers-res\"\>[^\<]{1,}/
-              ).to_s.slice(33, 999)
-            ) 
-                        
-          mylist =
-            Convert.commaRemover(
-              html.slice(
-                /<strong\sclass\=\"nico-numbers-mylist\"\>[^\<]{1,}/
-              ).to_s.slice(36, 999)
-            ) 
+          
+          /(<p\sclass\=\"nico-description\"\>)([^\<]{1,})/ =~ html
+          description = $2
+
+          /(<p\sclass\=\"nico-info-length\"\>)([^\<]{1,})/ =~ html
+          length = $2
+
+          /(<p\sclass\=\"nico-info-date\"\>)([^\<]{1,})/ =~ html
+          first_retrieve = $2
+
+          /(<p\sclass\=\"nico-numbers-view\"\>)([^\<]{1,})/ =~ html
+          view = $2
+
+          /(<p\sclass\=\"nico-numbers-res\"\>)([^\<]{1,})/ =~ html
+          res = $2
+
+          /(<p\sclass\=\"nico-numbers-mylist\"\>)([^\<]{1,})/ =~ html
+          mylist = $2
           
           parsed["entry"][n]["memo"] = memo 
           parsed["entry"][n]["thumbnail_url"] = thumbnail_url 
@@ -238,6 +216,7 @@ module NicoParser
     end
 
     doc.close 
+
     parsed
   end
     

data/lib/searcher.rb

@@ -1,4 +1,6 @@
-# -*- encoding: utf-8 -*-
+# -*- encoding: utf-8 -*-# -*- encoding: utf-8 -*-
+$:.unshift File.dirname(__FILE__) 
+
 require 'rubygems'
 require 'ruby-debug'
 
@@ -6,65 +8,13 @@ require 'time'
 require 'mechanize'
 require 'kconv'
 
-require 'parser'
-
-
-$wait_byTag = {
-  'consec_count'  => 10,  # 連続してリクエストする回数
-  'consec_wait'   => 10,  # 連続リクエスト後のウェイト
-  'each'          => 10,  # 連続リクエスト時の、1リクエスト毎のウェイト
-   
-  'rejected'      => 120, # アクセス拒絶時（「短時間での連続アクセスは・・・」）
-                          # の場合の再試行までの時間
-  '403'           => 600, # "403"時の再試行までのウェイト
-  'increment'     => 1,   # アクセス拒絶時の、次回以降の1リクエスト毎のウェイトの増加量
-
-  'timeout'       => 5,   # タイムアウト時の、再試行までのウェイト
-  '500'           => 600, # "500"時の再試行までのウェイト
-  '503'           => 600, # "503"時の再試行までのウェイト
-
-  'allowance_time'=> 5    # 再試行回数の限度
-}
-
-$wait_byMylistLt = {
-  'consec_count'  => 10, 
-  'consec_wait'   => 10, 
-  'each'          => 10, 
+require 'parser.rb'
 
-  'rejected'      => 120,   
-  '403'           => 600,   
-  'increment'     => 1,   
-  'timeout'       => 5,    
-  '500'           => 600, 
-  '503'           => 600, 
-  'allowance_time'=> 5  
-}
 
-module GetMovie   
-  public
-
-  def byTag (tag, sort, waitObj, &block)
-    gMByTag = GetMovieByTag.new()
-    gMByTag.execute(tag, sort, waitObj) { |result, page|
-      block.call(result, page)
-    }
-  end
-
-  def byTagLt (tag, sort, waitObj, &block)
-    gMByTagLt = GetMovieByTagLt.new()
-    gMByTagLt.execute(tag, sort, waitObj) { |result, page|
-      block.call(result, page)
-    }
-  end
-    
-  module_function :byTag
-  module_function :byTagLt
-end
-
-class GetMovieByTagSuper
+class SearchByTagSuper
   private
 
-  def get (tag, sort, page, method, waitObj)
+  def get(tag, sort, page, method, waitObj)
     paramAry = []
     
     case sort
@@ -94,7 +44,7 @@ class GetMovieByTagSuper
         sortStr = 'sort=l&order=a'
     end
     
-    if page != 1 then paramAry.push("page=#{page}"); end
+    paramAry.push("page=#{page}") if page != 1
     paramAry.push(sortStr)
     if method == "atom" then paramAry.push("rss=atom&numbers=1") end
     param = tag + "?" + paramAry.join('&')
@@ -102,18 +52,16 @@ class GetMovieByTagSuper
     host = 'www.nicovideo.jp'
     entity = '/tag/' + param
 
-    @con.setWait(waitObj)
-    @con.get(host, entity)
+    @connector.setWait(waitObj)
+    @connector.get(host, entity)
   end
 
-  public
-
-  def loop (tag, sort, method, waitObj, &block)
-    termFlag = false
-    page   = 1
+  def loop(tag, sort, method, waitObj, &block)
+    termFlag    = false
+    page        = 1
+    movieObjAry = []
    
     begin
-       result  = []
        response = get(
         tag,
         sort,
@@ -122,9 +70,16 @@ class GetMovieByTagSuper
         waitObj
       )
 
-      if response
-        result = parse(response)
-        termFlag = block.call(result, page)
+      if response["order"] == "success"
+        result = parse(response["body"])
+        result.each { |each|
+          movie = Movie.new(each["video_id"]) 
+          each["available"] = true
+          movie.set(each)
+          movieObjAry.push(movie)
+        }
+
+        termFlag = block.call(movieObjAry, page)
       else
         termFlag = true
       end
@@ -134,13 +89,12 @@ class GetMovieByTagSuper
   end
 end
 
-
-class GetMovieByTag < GetMovieByTagSuper
+class SearchByTag < SearchByTagSuper
   def initialize
-    @NumOfSearched = 32
+    @numOfSearched = 32
     @incrAmt = 0.2
 
-    @con = Connector.new('mech')
+    @connector = Connector.new('mech')
     
     # HTML中の各パラメータの所在を示すXPath
     @videoIdXP  = "//div[@class='uad_thumbfrm']/table/tr/td/p/a"
@@ -154,16 +108,16 @@ class GetMovieByTag < GetMovieByTagSuper
   def parse(movieNum)
     result = []
     
-    video_id  = /(sm|nm)[0-9]{1,}/.match(@con.mech.page.search(@videoIdXP)[movieNum]['href'])[0]
-      lengthStr = @con.mech.page.search(@lengthXP)[movieNum].text.split(/\:/)
+    video_id  = /(sm|nm)[0-9]{1,}/.match(@connector.mech.page.search(@videoIdXP)[movieNum]['href'])[0]
+      lengthStr = @connector.mech.page.search(@lengthXP)[movieNum].text.split(/\:/)
     length    = lengthStr[0].to_i * 60 + lengthStr[1].to_i
-    view      = @con.mech.page.search(@viewXP)[movieNum]
+    view      = @connector.mech.page.search(@viewXP)[movieNum]
                 .text.gsub(/\,/, '').to_i
-    res       = @con.mech.page.search(@resXP)[movieNum]
+    res       = @connector.mech.page.search(@resXP)[movieNum]
                 .text.gsub(/\,/, '').to_i
-    mylist    = @con.mech.page.search(@mylistXP)[movieNum]
+    mylist    = @connector.mech.page.search(@mylistXP)[movieNum]
                 .text.gsub(/\,/, '').to_i
-    ad        = @con.mech.page.search(@adXP)[movieNum]
+    ad        = @connector.mech.page.search(@adXP)[movieNum]
                 .text.gsub(/\,/, '').to_i
 
     result.push({
@@ -183,11 +137,11 @@ class GetMovieByTag < GetMovieByTagSuper
   end
 end
 
-class GetMovieByTagLt < GetMovieByTagSuper
+class SearchByTagLt < SearchByTagSuper
   def initialize
-    @NumOfSearched = 32
+    @numOfSearched = 32
     @incrAmt = 0.2
-    @con = Connector.new('atom')
+    @connector = SearchByTagAtomConnector.new()
   end
 
   def parse(xml)
@@ -199,7 +153,4 @@ class GetMovieByTagLt < GetMovieByTagSuper
       block.call(result, page)
     }
   end
-end
-
-
-
+end

data/nicoscraper.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{nicoscraper}
-  s.version = "0.1.0"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = [%q{Masami Yonehara}]
-  s.date = %q{2011-09-21}
+  s.date = %q{2011-09-22}
   s.description = %q{It scrape movies and mylists of Niconico douga.
   }
   s.email = %q{zeitdiebe@gmail.com}
@@ -34,6 +34,7 @@ Gem::Specification.new do |s|
     "lib/searcher.rb",
     "nicoscraper.gemspec",
     "test/helper.rb",
+    "test/movie_spec.rb",
     "test/test_nicoscraper.rb"
   ]
   s.homepage = %q{http://github.com/hdemon/nicoscraper}

data/test/movie_spec.rb

@@ -0,0 +1,125 @@
+# -*- encoding: utf-8 -*-
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'movie.rb'
+require 'mylist.rb'
+require 'searcher.rb'
+
+describe Movie, "After executiton of 'getInfo' method" do  
+  before(:all) do
+    @movie = Movie.new("sm1097445")
+    @movie.getInfo
+  end
+
+  it "should have following values" do
+    @movie.available    .should be_true
+  
+    @movie.video_id     .should == "sm1097445"
+    @movie.title        .should == "【初音ミク】みくみくにしてあげる♪【してやんよ】"
+    @movie.description  .should_not be_nil
+    @movie.thumbnail_url.should_not be_nil
+    @movie.first_retrieve.should == 1190218922
+    @movie.length       .should == 99
+    @movie.movie_type   .should == "flv"
+    @movie.size_high    .should == 3906547
+    @movie.size_low     .should == 1688098
+
+    @movie.view_counter .should_not be_nil
+    @movie.comment_num  .should_not be_nil
+    @movie.mylist_counter .should_not be_nil 
+    @movie.last_res_body.should_not be_nil 
+
+    @movie.watch_url    .should == "http://www.nicovideo.jp/watch/sm1097445"
+    @movie.thumb_type   .should == "video"
+    @movie.embeddable   .should == 1
+    @movie.no_live_play .should == 0
+    @movie.tags_jp      .should_not be_nil
+    @movie.tags_tw      .should_not be_nil
+    @movie.user_id      .should == 70391
+  end    
+
+  it "should return true when execute 'isBelongsTo' method." +
+      "passing arguments (1450136)" do
+    result = @movie.isBelongsTo(1450136)
+    result .should_not be_nil
+    result .should be_true
+  end
+end
+
+describe Movie, "when access with non-existent video_id" do
+  before(:all) do
+    @movie = Movie.new("sm*nonexistent")
+    @movie.getInfo
+  end
+
+  it "should be unavailable" do
+    @movie.available    .should be_false
+  end    
+end
+
+describe Mylist, "After executiton of 'getInfoLt' method" do  
+  before(:all) do
+    @mylist = Mylist.new(15196568)
+    @mylist.getInfoLt
+    puts @mylist
+  end
+
+  it "should have following values" do
+    @mylist.available    .should be_true
+  
+    @mylist.mylist_id    .should == 15196568
+    @mylist.user_id      .should be_nil
+    @mylist.title        .should == "【Oblivion】おっさんの大冒険"
+    @mylist.description  .should_not be_nil
+    @mylist.public       .should be_nil
+    @mylist.default_sort .should be_nil
+    @mylist.create_time  .should be_nil
+    @mylist.update_time  .should_not be_nil
+
+    @mylist.icon_id      .should be_nil
+    @mylist.movies       .should_not be_nil
+    @mylist.movies       .should be_kind_of(Array)
+    @mylist.author       .should == "おぽこ"
+  end 
+
+  it "should return over 0.9 when execute 'getSimilarity' method." +
+      "passing arguments (1450136)" do
+    result = @mylist.getSimilarity
+    result .should_not be_nil
+    result .should >= 0.9
+  end
+end
+
+describe "When execute 'SearchByTagLt.execute' method " +
+          "passing following argument" do  
+  before(:all) do
+    searcher = SearchByTagLt.new()
+    searcher.execute("ゆっくり実況プレイpart1リンク", "post_old", nil) { |result|
+      @result = result
+    }
+  end
+
+  it "should have Array of movie objects." do
+    @result   .should be_kind_of(Array)
+    @result[0].should be_instance_of(Movie)
+  end
+
+  it "should contains movie objects that have following structure." do
+    @result[0].available    .should be_true
+  
+    @result[0].video_id     .should_not be_nil
+    @result[0].title        .should_not be_nil
+    @result[0].published    .should_not be_nil
+    @result[0].updated      .should_not be_nil
+    #@result[0].memo         .should_not be_nil
+    @result[0].description  .should_not be_nil
+    @result[0].thumbnail_url.should_not be_nil
+    @result[0].published    .should_not be_nil
+    @result[0].updated      .should_not be_nil
+    @result[0].length       .should_not be_nil
+
+    @result[0].view_counter .should_not be_nil
+    @result[0].comment_num  .should_not be_nil
+    @result[0].mylist_counter.should_not be_nil 
+  end    
+end

metadata

@@ -2,7 +2,7 @@
 name: nicoscraper
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Masami Yonehara
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-21 00:00:00 Z
+date: 2011-09-22 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: damerau-levenshtein
@@ -104,6 +104,7 @@ files:
 - lib/searcher.rb
 - nicoscraper.gemspec
 - test/helper.rb
+- test/movie_spec.rb
 - test/test_nicoscraper.rb
 homepage: http://github.com/hdemon/nicoscraper
 licenses: 
@@ -118,7 +119,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 4166593295240346995
+      hash: -4040185051160948255
       segments: 
       - 0
       version: "0"