nicoscraper 0.1.0

data/lib/movie.rb

@@ -0,0 +1,297 @@
+# -*- encoding: utf-8 -*-
+require 'rubygems'
+require 'ruby-debug'
+require 'damerau-levenshtein'
+require 'kconv'
+
+require 'parser'
+require 'mylist'
+require 'connector'
+  
+class Movie  
+  def initialize(video_id)
+    @video_id   = video_id
+    @available  = false
+  end
+  
+  private
+   
+  public
+  
+  # 指定されたマイリストに自分が入っていれば、真を返す。  
+  def isBelongsTo (_mylistId, &block)
+    isBelongs = false
+    thisMl = Mylist.new(_mylistId)
+    thisMl.getInfoLt
+    
+    thisMl.movies.each { |movie|
+      isBelongs = true if movie.video_id == @video_id
+    }   
+  
+    if isBelongs
+      puts "\sThis movie is found in mylist/" + _mylistId
+    else
+      puts "\sThis movie is not found in mylist/" + _mylistId
+    end
+    
+    # 無駄なアクセスを省くため、マイリスト中の動画に関する追加処理があれば、
+    # ブロックとして実行できる。
+    block.call(thisMl)
+    
+    return isBelongs
+  end
+  
+  # 自分が含まれる、投稿者の作ったシリーズとしてまとめているマイリストのIDを返す。
+  # 情報取得元が異なるため、必ずしもisBelongsの結果とは包含関係にならない。
+  def isSeriesOf
+    if !@available then
+      puts "This movie object is not available."
+      return "failed"
+    end
+    
+    puts
+    puts "Start to discern the seriality of..."
+    puts "\svideo_id:\s\s" + @video_id
+    puts "\stitle:\s\s\s\s\s" + @title    
+    # extrMylist呼び出し
+    mylistIdAry = extrMylist
+    sMylistIdAry = []
+    mlObjAry = []
+    mylistId = nil
+    mylist = nil
+    similarity = 0.0
+    
+    mylistIdAry.each { |_mylistId|
+      belongsTo = isBelongsTo(_mylistId) { |mylistObj|        
+        similarity = mylistObj.isSeries 
+        puts "\sSimilarity:\t" + similarity.to_s
+      }
+      puts belongsTo
+      if belongsTo && similarity > 0.7
+        puts "\s" + _mylistId.to_s + "\tis perecieved as series mylist."
+        sMylistIdAry.push(_mylistId)
+      end  
+    }
+    
+    sMylistIdAry.each { |mylistId|
+      puts mylistId
+      mlObjAry.push( Mylist.new(mylistId) )
+    }
+    
+    puts "\sDiscern logic terminated."
+    return mlObjAry   
+  end
+
+  # 動画説明文中から、マイリストIDを示す文字列を抜き出す。
+  def extrMylist
+    return if !@available
+    puts "Extracting mylistId from the description..."
+    
+    mylistIdAry = []
+    extracted = @description.scan(/mylist\/[0-9]{1,8}/)
+    if extracted[0] != nil
+      extracted.each { |e|
+        id = e.scan(/[0-9]{1,8}/)[0]
+        mylistIdAry.push(id)
+        puts "\sID:\t" + id + " is extracted."
+      }
+    else
+      puts "\sMylistId is not found."
+    end  
+    
+    return mylistIdAry
+  end
+  
+  def getInfo
+    con = Connector.new('xml')
+    host = 'ext.nicovideo.jp'
+    entity = '/api/getthumbinfo/' + @video_id
+    con.setWait(nil)
+    xml = con.xmlGet(host, entity)
+    
+    unless
+      xml =~ /<nicovideo_thumb_response\sstatus=\"fail\">/ ||
+      xml == "failed"
+    then
+      param = NicoParser.getThumbInfo(xml)
+      set(param)
+      @available = true
+    else
+      @available = false
+      return "failed"
+    end
+  end
+    
+  def set(paramObj)
+    paramObj.each_key { |key|
+      param = paramObj[key]
+      case key
+      when "available"
+        @available = param
+        
+      when "video_id"
+        @video_id = param
+      when "mylist_id"    
+        @mylist_id = param
+    	when "item_id"
+        @item_id = param
+    	when "description"
+        @description = param
+      
+      # MylistAPI
+      when "video_id"
+        @video_id = param
+    	when "item_id"
+        @item_id = param.to_i
+    	when "description"
+        @description = param        
+      when "item_data"
+        paramObj['item_data'].each_key { |key|
+        param = paramObj['item_data'][key]
+          case key
+          when "video_id"
+            @video_id = param   
+        	when "title"
+            @title = param
+        	when "thumbnail_url"
+            @thumbnail_url = param
+        	when "first_retrieve"
+            @first_retrieve = param
+        	when "update_time"
+            @update_time = param
+        	when "view_counter"
+            @view_counter = param.to_i
+        	when "mylist_counter"
+            @mylist_counter = param.to_i
+        	when "num_res"
+            @comment_num = param.to_i
+        	when "length_seconds"
+            @length = param
+        	when "deleted"
+            @deleted = param.to_i       
+        	when "last_res_body"
+            @last_res_body = param
+          end
+        } 
+    	when "watch"
+        @watch = param
+    	when "create_time"
+        @create_time = param
+    	when "update_time"
+        @update_time = param
+      
+      # MylistAPI-Atom
+      when "video_id"
+        @video_id = param
+    	when "item_id"
+        @item_id = param
+    	when "memo"
+        @memo = param        
+    	when "published"
+        @published = param       
+    	when "updated"
+        @updated = param    
+    	when "thumbnail_url"
+        @thumbnail_url = param     
+    	when "length"
+        @length = param
+    	when "view"
+        @view_counter = param.to_i
+    	when "mylist"
+        @mylist_counter = param.to_i
+    	when "res"
+        @comment_num = param.to_i     
+    	when "first_retrieve"
+        @first_retrieve = param       
+    	when "length"
+        @length = param        
+      
+      # getThumbInfo  
+      when "video_id"
+        @video_id = param   
+    	when "title"
+        @title = param
+    	when "description"
+        @description = param  
+    	when "thumbnail_url"
+        @thumbnail_url = param
+    	when "first_retrieve"
+        @first_retrieve = param
+    	when "length"
+        @length = param
+    	when "movie_type"
+        @movie_type = param
+      when "size_high"
+        @size_high = param
+    	when "size_low"
+        @size_low = param
+    	when "view_counter"
+        @view_counter = param
+    	when "mylist_counter"
+        @mylist_counter = param
+    	when "comment_num"
+        @comment_num = param
+    	when "last_res_body"
+        @last_res_body = param
+    	when "watch_url"
+        @watch_url = param
+    	when "thumb_type"
+        @thumb_type = param
+    	when "embeddable"
+        @embeddable = param
+    	when "movieNum_live_play"
+        @movieNum_live_play = param
+    	when "tags_jp"
+        @tags_jp = param
+      when "tags_tw"
+        @tags_tw = param
+    	when "tags_de"
+        @tags_de = param
+      when "tags_sp"
+        @tags_sp = param
+    	when "user_id"
+        @user_id = param
+      end
+    }   
+  end  
+  
+  attr_accessor :available
+  
+  # MylistAPI   
+  attr_accessor	:video_id
+  attr_accessor	:mylist_id
+  attr_accessor	:item_id
+  attr_accessor	:description
+
+  attr_accessor	:title
+  attr_accessor	:thumbnail_url
+  attr_accessor	:first_retrieve
+  attr_accessor	:update_time
+  attr_accessor	:view_counter
+  attr_accessor	:mylist_counter
+  attr_accessor	:comment_num
+  attr_accessor	:length
+  attr_accessor	:deleted
+  attr_accessor	:last_res_body
+
+  attr_accessor	:watch
+  attr_accessor	:create_time
+  attr_accessor	:update_time
+
+  # MylistAPI-Atom
+  attr_accessor :memo
+  attr_accessor	:published
+  attr_accessor	:updated
+  
+  # getThumbInfo
+  attr_accessor	:movie_type
+  attr_accessor	:size_high
+  attr_accessor	:size_low
+  attr_accessor	:watch_url
+  attr_accessor	:thumb_type
+  attr_accessor	:embeddable
+  attr_accessor	:movieNum_live_play
+  attr_accessor	:tags_jp
+  attr_accessor	:tags_tw
+  attr_accessor	:user_id
+end

data/lib/mylist.rb

@@ -0,0 +1,258 @@
+# -*- encoding: utf-8 -*-
+require 'rubygems'
+require 'ruby-debug'
+require 'kconv'
+
+require 'parser'
+require 'movie'
+require 'connector'
+
+
+class Mylist
+  def initialize (mylist_id)
+    @mylist_id  = mylist_id    
+    @movies     = []
+    @available  = false
+  end
+  
+  def simOfTitle  
+    match = false
+    dl = DamerauLevenshtein
+    d = 0.0
+    
+    puts "matching..."
+    
+    # O(n^2)なのでどうにかしたい。しかし、最大でも500C2=125000なので、
+    # 日々の利用については許容できると思う、
+    @movies.each { |myself|
+      @movies.each { |amovieNumther|
+        _d = dl.distance(myself.title, amovieNumther.title)
+        d += _d
+      }
+    }
+    
+    similarity = 1 - ( (d / @movies.length) / title.length )
+    puts "Similarity: " + similarity.to_s
+    similarity
+  end 
+
+  def userId
+    idGroup = {}
+    samePublisher = false
+    threshold = 0.9
+    
+    @movies.each { |movie|
+      idGroup[movie.user_id] += 1
+    }
+    
+    idGroup.each { |group|
+      if @movies.length / threshold < group.length
+        samePublicher = true  
+      end
+    }
+    
+    return samePublisher    
+  end
+  
+  # 自分がシリーズをまとめたマイリストであるかを判定する。
+  # 判定基準は、1.一定数以上の動画の投稿者が、マイリスト作成者と同じであること。
+  # 2.タイトルの類似度が、定められた基準以上であること。
+  def isSeries
+    l = @movies.length - 1
+    dlc = DamerauLevenshtein
+    dl = 0.0
+    dlAry = []
+    count_o = 0
+    count_i = 0
+    
+    while count_o <= l do
+      count_i = count_o + 1
+      while count_i <= l do
+        dl = dlc.distance(
+          @movies[count_i].title, 
+          @movies[count_o].title
+        )
+        
+        dl = 1.0 - dl.fdiv( @movies[count_i].title.length)
+        dlAry.push(dl)
+        
+        count_i += 1
+      end
+      count_o += 1
+    end
+    
+    if l != 0 && dlAry.length > 0
+      t = 0
+      dlAry.each { |_dl| t += _dl }
+      similarity = t / dlAry.length
+    elsif dlAry.length == 0
+      similarity = 0
+    else
+      similarity = 1
+    end
+        
+    return similarity
+  end
+
+  def getInfo
+    con = Connector.new('mech')
+    reqUrl = 'http://www.nicovideo.jp' +
+      '/mylist/' + @mylist_id.to_s
+    con.setWait(nil)
+    mechPage = con.mechGet(reqUrl)
+    result = []
+
+    # Mylist自身の情報を取得    
+    jsonStr = mechPage.search(
+      "/html/body/div[2]" +
+      "/div/div[2]/script[7]"
+    ).to_html
+    
+    reg = /MylistGroup\.preloadSingle.{1,}?Mylist\.preload\(/m
+    mlJson = jsonStr.scan(reg)[0]
+      
+    id = mlJson.scan(/\sid:[^\n]{1,}/)[0]
+        .scan(/[0-9]{1,}/)[0]
+    user_id = mlJson.scan(/\suser_id:[^\n]{1,}/)[0]
+        .scan(/[0-9]{1,}/)[0]
+    name = mlJson.scan(/\sname:[^\n]{1,}/)[0]
+      name = name.slice(
+        " name: \"".length,
+        name.length - " name: \"".length - "\",\n".length
+      )
+    desc = mlJson.scan(/\sdescription:.{1,}/)[0]
+      desc = desc.slice(
+        " description: \"".length,
+        desc.length - " description: \"".length - "\",\npublic".length
+      )
+    public = mlJson.scan(/\spublic:[^,]{1,}/)[0]
+        .scan(/[0-9]{1,}/)[0]
+    default_sort = mlJson.scan(/\sdefault_sort:[^\n]{1,}/)[0]
+        .scan(/[0-9]{1,}/)[0]
+    create_time = mlJson.scan(/\screate_time:[^\n]{1,}/)[0]
+        .scan(/[0-9]{1,}/)[0]
+    update_time = mlJson.scan(/\supdate_time:[^\n]{1,}/)[0]
+        .scan(/[0-9]{1,}/)[0]
+    icon_id = mlJson.scan(/\sicon_id:[^\n]{1,}/)[0]
+        .scan(/[0-9]{1,}/)[0]
+ 
+   # mlJson = mlJson.scan(/[^\r\n  ]{1,}/).join('')
+    #mlJson = mlJson.scan(/{.+/)[0].split(',')
+    
+    # 説明文が空欄だった時の措置。
+    desc   = mlJson[3].scan(/\".+\"/)[0]
+    if desc != nil then desc = desc.scan(/[^\"]{1,}/)[0] end
+
+    paramObj = {
+      "id"            => id,
+      "user_id"       => user_id,
+      "name"          => name,
+      "description"   => description,
+      "public"        => public,
+      "default_sort"  => default_sort,
+      "create_time"   => create_time,
+      "update_time"   => update_time,
+      "icon_id"       => icon_id
+      # "sort_order"  => ,
+    }
+    set(paramObj)  
+    
+    # 自分に含まれる動画の情報を取得
+    jsonStr = mechPage.search(
+      "/html/body/div[2]" +
+      "/div/div[2]/script[7]"
+    ).to_html
+
+    mvJson = jsonStr.scan(/Mylist.preload.+/)[0]
+    mvJson = mvJson.scan(/\".{1,}/)[0]
+    mvJson = mvJson.slice(0, mvJson.length - 5)
+    #mvJson = mvJson.split('},{')
+    mvJson = Unicode.unescape(mvJson).split('},{')
+    
+    mvJson.each { |e|
+      e = "{" + e + "}"
+      param = JSON.parse(e) 
+      movie = Movie.new(param['item_data']['video_id'])
+      movie.set(param)
+      
+      @movies.push(movie)
+    }    
+  end
+  
+  def getInfoLt
+    con = Connector.new('xml')
+    host = 'www.nicovideo.jp'
+    puts @mylist_id
+    entity = '/mylist/' + @mylist_id.to_s + '?rss=atom&numbers=1'
+    con.setWait(nil)
+    xml = con.xmlGet(host, entity)
+    
+    unless
+      xml == "failed"
+    then  
+      parsed = NicoParser.mylistRss(xml)
+      
+      parsed["entry"].each { |e|
+        movie = Movie.new(e["video_id"])
+        e["available"] = true
+        movie.set(e)
+        @movies.push(movie)
+      }
+      
+      set(parsed["mylist"])
+      @available = true
+    end    
+  end  
+
+  def set(paramObj)
+    paramObj.each_key { |key|
+      param = paramObj[key]
+      case key
+      when "mylist_id"    
+        @mylist_id = param
+      when "id"    
+        @mylist_id = param
+    	when "user_id"
+        @user_id = param
+    	when "title"
+        @title = param
+    	when "description"
+        @description = param
+    	when "public"
+        @public = param
+    	when "default_sort"
+        @default_sort = param
+    	when "create_time"
+        @create_time = param
+    	when "update_time"
+        @update_time = param
+    	when "icon_id"
+        @icon_id = param
+    	when "sort_order"
+        @sort_order = param
+    	when "movies"
+        @movies = param
+
+    	when "updated"
+        @update_time = param
+    	when "author"
+        @author = param
+      end
+    }   
+  end
+
+  attr_accessor :available
+    
+  attr_accessor :mylist_id    
+	attr_accessor :user_id    
+	attr_accessor :title       
+	attr_accessor :description  
+	attr_accessor :public     
+	attr_accessor :default_sort   
+	attr_accessor :create_time  
+	attr_accessor :update_time  
+	attr_accessor :icon_id    
+	attr_accessor :sort_order   
+  
+  attr_accessor :movies     
+end