emergent-nicovideo 0.1.7

data/lib/nicovideo/page.rb

@@ -0,0 +1,92 @@
+require 'kconv'
+
+module Nicovideo
+
+  class Page
+    NV_DEBUG_LEVEL = 0
+
+    BASE_URL = 'http://www.nicovideo.jp'
+    BASE_TITLE1  = '‐ニコニコ動画\(秋\)'.toutf8
+    BASE_TITLE2  = 'ニコニコ動画\(秋\)‐'.toutf8
+
+    def initialize agent
+      @agent = agent
+      @page  = nil
+      @title = nil
+
+      @not_found = false
+    end
+    
+    public
+    def exists?()
+      begin
+        @page = @page || get_page
+        return true
+      rescue
+        return false
+      end
+    end
+
+    def html()
+      page = @page || get_page
+      return nil unless page
+      page.parser.to_html
+    end
+
+    def title=(title)
+      @title = title
+    end
+
+    protected
+    def register_getter(params)
+      params.each {|p|
+        p_noq = p.sub(/\?$/,'')
+        eval <<-E
+          @#{p_noq} = nil
+          def #{p}
+            if @#{p_noq}.nil?
+              @page ||= get_page(@url)
+            end
+            @#{p_noq}
+          end
+        E
+      }
+    end
+    
+    def parse page
+      # to be extended
+    end
+
+    def get_page url, force=false
+      return @page if (@page && !force) 
+      raise NotFound if @not_found
+
+      puts_info 'getting html page : url = ' + url.to_s
+      begin
+        page = @agent.get(url)
+        puts_debug page.header
+        puts_debug page.body
+
+        parse(page)
+        @page = page
+      rescue WWW::Mechanize::ResponseCodeError => e
+        rc = e.response_code
+        puts_info rc
+        if rc == "404" || rc == "410"
+          @not_found = true
+          raise NotFound
+        elsif rc == "403"
+          raise Forbidden
+        else
+          raise e
+        end
+      end
+      @page
+    end
+
+    def puts_error str ; puts str if (NV_DEBUG_LEVEL >= 1) ; end
+    def puts_info  str ; puts str if (NV_DEBUG_LEVEL >= 2) ; end
+    def puts_debug str ; puts str if (NV_DEBUG_LEVEL >= 3) ; end
+  end
+
+end

data/lib/nicovideo/random.rb

@@ -0,0 +1,38 @@
+module Nicovideo
+  class Random < Page
+    include Enumerable
+
+    def initialize agent
+      super(agent)
+      @url = url()
+      self.register_getter ["videos"]
+    end
+
+    def each
+      self.videos.each {|v| yield v }
+    end
+
+    def url
+      "#{BASE_URL}/random"
+    end
+
+    def to_a
+      videos()
+    end
+
+    def reload
+    end
+
+    protected
+    def parse(page)
+      result_xpath = page/'//td[@class="random_td"]//p[@class="TXT12"]/a[@class="video"]'
+      @videos = result_xpath.inject([]) {|arr,v| #
+        #puts v.attributes['href']
+        vp = VideoPage.new(@agent, v.attributes['href'].sub(/watch\/(\w+)$/,'\1'))
+        vp.title = v.inner_html
+        arr << vp
+      }
+    end
+
+  end
+end

data/lib/nicovideo/ranking.rb

@@ -0,0 +1,36 @@
+module Nicovideo
+  class Ranking < Page
+    def initialize agent, type='mylist', span='daily', category='all', pagenum=nil
+      super(agent)
+      @type     = type
+      @category = category
+      @pagenum  = pagenum
+      @url      = url()
+      self.register_getter ["videos"]
+    end
+
+    def url
+      url = "#{BASE_URL}/ranking/#{@type}/#{@span}/#{@category}"
+      if @pagenum
+        url += '?page=' + @pagenum.to_s
+      end
+      url
+    end
+
+    def to_a
+      videos()
+    end
+
+    protected
+    def parse(page)
+      ranking = page/'h3/a[@class=video]'
+      @videos = ranking.inject([]) {|arr,v| #
+        #puts v.attributes['href']
+        vp = VideoPage.new(@agent, v.attributes['href'].sub(/#{BASE_URL}\/watch\/(\w+)$/,'\1'))
+        vp.title = v.inner_html
+        arr << vp
+      }
+    end
+
+  end
+end

data/lib/nicovideo/search.rb

@@ -0,0 +1,100 @@
+require 'cgi'
+
+module Nicovideo
+  class Search < Page
+    include Enumerable
+
+    def initialize agent, keyword, sort=nil, order=nil, pagenum=1
+      super(agent)
+      @search_type = 'search'
+      #@keyword = CGI.escape(CGI.escape(keyword))
+      @keyword = CGI.escape(keyword)
+      @sort    = sort
+      @order   = order
+      @pagenum = pagenum
+
+      params = ["videos", "total_size", "has_next?", "has_prev?"]
+      self.register_getter params
+
+      @url = url()
+
+      puts_info "url = #{@url}"
+      puts_info "sort=#{@sort},order=#{@order},pagenum=#{@pagenum}"
+    end
+
+    def url
+      url = "#{BASE_URL}/#{@search_type}/#{@keyword}"
+      url += '?' if (@sort || @order || @pagenum)
+      url += '&sort='  + @sort  if @sort
+      url += '&order=' + @order if @order
+      url += '&page=' + @pagenum.to_s if @pagenum
+      url
+    end
+
+    def each
+      self.videos.each {|v|
+        yield v
+      }
+    end
+
+    def to_a() self.videos end
+
+    def pagenum=(pagenum)
+      if @pagenum != pagenum
+        @pagenum = pagenum
+        get_page(self.url, true)
+      end
+      @pagenum
+    end
+
+    def page=(pagenum)
+      self.pagenum = pagenum
+    end
+
+    def next
+      self.pagenum = @pagenum + 1
+      self
+    end
+
+    def prev
+      self.pagenum = @pagenum - 1
+      self
+    end
+
+    protected
+    def parse(page)
+      if page.body =~ /<\/strong> を含む動画はありません。/
+          @not_found = true
+          raise NotFound
+      end
+
+      @total_size = page.search('form[@name="sort"]//td[@class="TXT12"]//strong').first.inner_html.sub(/,/,'').to_i
+
+      @has_next = false
+      @has_prev = false
+      respages = page/'//div[@class="mb16p4"]//p[@class="TXT12"]//a'
+      puts_info respages.size
+      respages.each {|r| puts_info r.inner_html }
+      if respages.size > 0
+        respages.each {|text|
+          if text.inner_html =~ /前のページ/
+            @has_prev = true
+          end
+          if text.inner_html =~ /次のページ/
+            @has_next = true
+          end
+        }
+      end
+
+      result_xpath = page/'//div[@class="cmn_thumb_R"]//p[@class="TXT12"]/a[@class="video"]'
+      
+      puts_info result_xpath.size.to_s
+      @videos = result_xpath.inject([]) {|arr, v|
+        vp = VideoPage.new(@agent, v.attributes['href'].sub(/watch\/(\w+)$/,'\1'))
+        vp.title = v.inner_html
+        arr << vp
+      }
+    end
+    
+  end
+end

data/lib/nicovideo/tagsearch.rb

@@ -0,0 +1,13 @@
+require 'cgi'
+
+module Nicovideo
+  class TagSearch < Search
+
+    def initialize agent, keyword, sort=nil, order=nil, pagenum=1
+      super(agent, keyword, sort, order, pagenum)
+      @search_type = 'tag'
+      @url = url()
+    end
+
+  end
+end

data/lib/nicovideo/thumbnail.rb

@@ -0,0 +1,68 @@
+require 'open-uri'
+require 'timeout'
+require 'rexml/document'
+
+module Nicovideo
+	class Thumbnail
+    def initialize(proxy_url = nil)
+      @proxy_url =  proxy_url
+    end
+		
+		def get(video_id, wait_sec = 10, retry_max = 2)
+			root = get_response(video_id, wait_sec, retry_max)
+			
+			get_elements(root.elements["thumb"])
+		end
+
+		def get_response(video_id, wait_sec, retry_max)
+			retry_count = 0
+      begin
+        body = timeout(wait_sec) do
+          open("http://ext.nicovideo.jp/api/getthumbinfo/#{video_id}", :proxy => @proxy_url) do |f|
+            f.read
+          end
+        end
+
+        root = REXML::Document.new(body).root
+        raise ::Errno::ENOENT::new(video_id) unless root.attributes.get_attribute('status').value == 'ok'
+        root
+      rescue TimeoutError => e
+        raise e if retry_count >= retry_max
+        retry_count += 1
+        retry
+      end
+		end
+		
+		def get_elements(parent)
+			thumbnail_info = ThumbInfo.new
+
+			parent.each_element do |element|
+				if element.name == 'tags' then
+          thumbnail_info.tags[element.attributes['domain']] = []
+          element.each_element do |child|
+            thumbnail_info.tags[element.attributes['domain']] << child.text
+          end
+					next
+			  end
+				thumbnail_info[element.name] = element.text
+			end
+  		thumbnail_info
+		end
+	end
+
+  class ThumbInfo < Hash
+    attr_accessor :tags
+    def initialize
+      @tags = {}
+    end
+
+    def has_tag?(tag)
+      @tag_hash ||= tag_flatten.inject({}) {|tag_hash, temp_tag| tag_hash[temp_tag] = temp_tag}
+      @tag_hash.has_key? tag
+    end
+
+    def tag_flatten
+      @tag_flatten ||= @tags.values.flatten
+    end
+  end
+end

data/lib/nicovideo/version.rb

@@ -0,0 +1,9 @@
+module Nicovideo #:nodoc:
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 1
+    TINY  = 7
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/lib/nicovideo/videopage.rb

@@ -0,0 +1,113 @@
+require 'kconv'
+require 'cgi'
+
+module Nicovideo
+  class VideoPage < Page
+    def initialize agent, video_id
+      super(agent)
+      @video_id = video_id
+      @params   = nil
+      @url      = BASE_URL + '/watch/' + @video_id
+      register_getter ["title", "tags", "published_at", "csrf_token"]
+    end
+
+    attr_reader :video_id, :url
+
+    def id()   @video_id end
+
+    def type
+      @params ||= get_params
+      pattern = %r!^http://.*\.nicovideo\.jp/smile\?(.*?)=.*$!
+      CGI.unescape(@params['url']) =~ pattern
+      case $1
+      when 'm'
+        return 'mp4'
+      when 's'
+        return 'swf'
+      else
+        return 'flv'
+      end
+    end
+
+    def comments(num=500)
+      puts_info 'getting comment xml : id = ' + @video_id
+      begin
+        @params = get_params unless @params
+        ms = @params['ms']
+        raise ArgError unless ms
+        
+        thread_id = @params['thread_id']
+        body = %!<thread res_from="-#{num}" version="20061206" thread="#{thread_id}" />!
+        post_url = CGI.unescape(ms)
+        comment_xml = @agent.post_data(post_url, body).body
+        puts_debug comment_xml
+        Comments.new(@video_id, comment_xml)
+      end
+    end
+
+    def flv() return video() end
+
+    def video()
+      begin
+        @params ||= get_params
+        video_url = CGI.unescape(@params['url'])
+        video_flv = @agent.get_file(video_url)
+        video_flv
+      end
+    end
+
+    def title=(title)
+      @title = title
+    end
+
+    def openlist(page=1)
+      OpenList.new(@agent, @video_id)
+    end
+
+    def low?
+      @params ||= get_params
+      return true if CGI.unescape(@params['url']) =~ /low$/
+      return false
+    end
+
+    private
+    def parse(page)
+      # title
+      @title = page.title.toutf8.sub(/#{BASE_TITLE1}$/ou, '')
+
+      # tags
+      div = page.parser.search("div#video_tags")
+      @tags = div.to_html.scan(/<a href=\"tag\/[\w\%]+?\">(.+?)<\/a>/ou).inject([]) {|arr, v|
+        puts_debug v[0]
+        arr << v[0]
+      }
+
+      # published_at
+      str = page.search("div[@id='WATCHHEADER']//p[@class='TXT12']/strong")[0].inner_text
+      tm = str.scan(/\d+/)
+      @published_at = Time.mktime(*tm)
+      
+      # csrf_token
+      @csrf_token = page.search("form[@name='mylist_form']//input[@name='csrf_token']")[0]['value']
+    end
+    
+    def get_params
+      raise NotFound if @not_found
+      begin
+        unless @params
+          puts_info 'getting params : id = ' + @video_id
+          @page ||= get_page(@url)
+          content = @agent.get_file(BASE_URL + '/api/getflv?v=' + @video_id)
+          puts_debug content
+          @params = content.scan(/([^&]+)=([^&]*)/).inject({}){|h, v| h[v[0]] = v[1]; h}
+        else
+          puts_info 'params have already gotten : id = ' + @video_id
+        end
+        @params
+      rescue
+        @not_found = true
+        raise NotFound
+      end
+    end
+  end
+end

data/lib/nicovideo.rb

@@ -0,0 +1,20 @@
+$:.unshift File.dirname(__FILE__)
+
+require 'rubygems'
+require 'nicovideo/mechanize-ext'
+require 'nicovideo/base'
+require 'nicovideo/page'
+require 'nicovideo/videopage'
+require 'nicovideo/comments'
+require 'nicovideo/mylist'
+require 'nicovideo/openlist'
+require 'nicovideo/search'
+require 'nicovideo/tagsearch'
+require 'nicovideo/ranking'
+require 'nicovideo/random'
+require 'nicovideo/newarrival'
+require 'nicovideo/thumbnail'
+#require 'nicovideo/tags'
+#require 'nicovideo/ichiba'
+#require 'nicovideo/feed'
+

data/sample/nv_download.rb

@@ -0,0 +1,42 @@
+require 'rubygems'
+require 'nicovideo'
+require 'yaml'
+
+video_ids = ARGV
+
+# set account
+account = YAML.load_file(ENV['HOME'] + '/.nicovideo/account.yml')
+mail = account['mail']
+password = account['password']
+
+# create instance
+nv = Nicovideo.new(mail, password)
+
+# login to Nicovideo (you don't need to login explicitly at v 0.0.4 or later)
+nv.login
+
+# get videos and comments
+video_ids.each {|video_id|
+
+  nv.watch(video_id) {|v|
+    # method 'id' and 'video_id' return video ID(string).
+    puts 'video id = ' + v.id
+
+    # method 'title' returns string.
+    puts 'title = ' + v.title
+
+    # method 'tags' returns array of string.
+    puts 'tags = ' + v.tags.join(' ')
+
+    # method 'comments' returns instance of class Comments
+    # which has methods 'to_xml', 'to_s'(same).
+    puts 'getting comments xml'
+    File.open("#{video_id}.xml", "wb") {|f| f.write v.comments(100).to_xml }
+
+    # method 'flv' and 'video' return raw flv data(binary).
+    puts 'getting flv file'
+    File.open("#{video_id}.flv", "wb") {|f| f.write v.flv }
+  }
+
+  sleep 3
+}

data/sample/nv_download2.rb

@@ -0,0 +1,34 @@
+require 'rubygems'
+require 'nicovideo'
+require 'yaml'
+
+video_ids = ARGV
+
+# set account
+account = YAML.load_file(ENV['HOME'] + '/.nicovideo/account.yml')
+mail = account['mail']
+password = account['password']
+
+# create instance
+nv = Nicovideo.new(mail, password)
+
+# login to Nicovideo (you don't need to login explicitly at v 0.0.4 or later)
+nv.login
+
+# get videos and comments
+video_ids.each {|video_id|
+
+  # the another way of nv_download
+  puts nv.get_title(video_id)
+  puts nv.get_tags(video_id).join(' ')
+  puts 'getting comments xml'
+  File.open("#{video_id}.xml", "wb") {|f|
+    f.write nv.get_comments(video_id, 100).to_xml
+  }
+  puts 'getting flv file'
+  File.open("#{video_id}.flv", "wb") {|f|
+    f.write nv.get_flv(video_id)
+  }
+
+  sleep 1
+}

data/sample/nv_mylist.rb

@@ -0,0 +1,37 @@
+require 'rubygems'
+require 'nicovideo'
+require 'yaml'
+
+mylist_ids = ARGV
+
+# set account
+account = YAML.load_file(ENV['HOME'] + '/.nicovideo/account.yml')
+mail = account['mail']
+password = account['password']
+
+# create instance
+nv = Nicovideo.new(mail, password)
+
+# login to Nicovideo (you don't need to login explicitly at v 0.0.4 or later)
+nv.login
+
+# get mylist
+mylist_ids.each {|mylist_id|
+
+  ml = nv.mylist(mylist_id)
+  # method 'id' and 'mylist_id' return mylist ID(string).
+  puts 'mylist id = ' + ml.id
+
+  # method 'title', 'user' and 'description' return string.
+  puts 'title = ' + ml.title
+  puts 'user = ' + ml.user
+  puts 'description = ' + ml.description
+  
+  # method 'videos' returns array of VideoPage.
+  videos = ml.videos
+  videos.each {|v|
+    puts v.id
+  }
+
+  sleep 3
+}

data/sample/nv_openlist.rb

@@ -0,0 +1,35 @@
+require 'rubygems'
+require 'nicovideo'
+require 'yaml'
+
+video_ids = ARGV
+
+# set account
+account = YAML.load_file(ENV['HOME'] + '/.nicovideo/account.yml')
+mail = account['mail']
+password = account['password']
+
+# create instance
+nv = Nicovideo.new(mail, password)
+
+# login to Nicovideo (you don't need to login explicitly at v 0.0.4 or later)
+nv.login
+
+# get openlist
+video_ids.each {|video_id|
+
+  ol = nv.openlist(video_id)
+  # method 'id' and 'video_id' return video ID(string).
+  puts 'video id = ' + ol.id
+
+  # method 'total_size' returns Fixnum.
+  puts 'total_size = ' + ol.total_size.to_s
+  
+  # method 'mylists' returns array of MyList.
+  begin
+    mls = ol.mylists
+    mls.each {|ml| puts ml.id + ':' + ml.title }
+    sleep 1
+  end while (ol.has_next? && ol.next)
+  
+}

data/sample/nv_ranking.rb

@@ -0,0 +1,21 @@
+#!/usr/bin/ruby -Ku
+
+require 'rubygems'
+require 'nicovideo'
+
+conf = YAML.load_file(ENV['HOME'] + '/.nicovideo/account.yml')
+nv = Nicovideo.new(conf['mail'], conf['password'])
+
+cnt = 0
+nv.ranking.each do |vp|
+  cnt += 1
+  printf "%3d位 %-12s %s \n", cnt, vp.video_id, vp.title
+end
+
+=begin
+# when using more options
+nv.ranking(type='mylist', span='daily', category='all', pagenum='1').each do |vp|
+  cnt += 1
+  printf "%3d位 %-12s %s \n", cnt, vp.video_id, vp.title
+end
+=end

data/test/runner.rb

@@ -0,0 +1,3 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+Test::Unit::AutoRunner.run(true, File.dirname(__FILE__))

data/test/test_helper.rb

@@ -0,0 +1,2 @@
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/nicovideo'