nicovideo 0.0.1

data/README.txt

@@ -0,0 +1,37 @@
+= Nicovideo
+
+Nicovideo is scraping NICONICO DOUGA utility for Ruby
+(NICONICO DOUGA is Japanese Video Sharing Service)
+
+- NICONICO DOUGA : http://www.nicovideo.jp/
+
+== Installation
+
+ $ sudo gem install nicovideo
+
+== Usage
+
+See sample/nv_download.rb
+
+== Author
+
+- emergent ( http://d.hatena.ne.jp/emergent )
+- thanks for many bloggers who wrote scraping scripts 
+
+== License
+
+- 2-clause BSD License
+
+
+
+
+
+
+
+
+
+
+
+
+
+

data/Rakefile

@@ -0,0 +1,4 @@
+require 'config/requirements'
+require 'config/hoe' # setup Hoe + all gem configuration
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }

data/config/hoe.rb

@@ -0,0 +1,71 @@
+require 'nicovideo/version'
+
+AUTHOR = 'emergent'  # can also be an array of Authors
+EMAIL = "emergent22 (at) livedoor.com"
+DESCRIPTION = "utils for nicovideo"
+GEM_NAME = 'nicovideo' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'nicovideo' # The unix name for your project
+HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+RUBYFORGE_USERNAME = "unknown"
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  RUBYFORGE_USERNAME.replace @config["username"]
+end
+
+
+REV = nil 
+# UNCOMMENT IF REQUIRED: 
+# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
+VERS = Nicovideo::VERSION::STRING + (REV ? ".#{REV}" : "")
+RDOC_OPTS = ['--quiet', '--title', 'nicovideo documentation',
+    "--opname", "index.html",
+    "--line-numbers", 
+    "--main", "README",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps 
+    @extra_deps.reject! { |x| Array(x).first == 'hoe' } 
+    @extra_deps
+  end 
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.author = AUTHOR 
+  p.description = DESCRIPTION
+  p.email = EMAIL
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.test_globs = ["test/**/test_*.rb"]
+  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
+  
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\\n\\n")
+  #p.extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
+  p.extra_deps = [
+                  ['mechanize', '>=0.6.0']
+                 ]
+  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+end
+
+CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')

data/config/requirements.rb

@@ -0,0 +1,17 @@
+require 'fileutils'
+include FileUtils
+
+require 'rubygems'
+%w[rake hoe newgem rubigen].each do |req_gem|
+  begin
+    require req_gem
+  rescue LoadError
+    puts "This Rakefile requires the '#{req_gem}' RubyGem."
+    puts "Installation: gem install #{req_gem} -y"
+    exit
+  end
+end
+
+$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))
+
+require 'nicovideo'

data/lib/nicovideo.rb

@@ -0,0 +1,11 @@
+$:.unshift File.dirname(__FILE__)
+
+require 'nicovideo/mechanize-ext'
+require 'nicovideo/base'
+require 'nicovideo/video'
+require 'nicovideo/comments'
+#require 'nicovideo/tags'
+#require 'nicovideo/ichiba'
+#require 'nicovideo/rank'
+#require 'nicovideo/feed'
+

data/lib/nicovideo/base.rb

@@ -0,0 +1,145 @@
+require 'cgi'
+require 'kconv'
+
+module Nicovideo
+
+  class NicovideoArgError      < StandardError ; end
+  class NicovideoLoginError    < StandardError ; end
+  class NicovideoNotFoundError < StandardError ; end
+
+  class Base
+    NV_DEBUG_LEVEL = 0
+    TITLE_NICONICO = 'ニコニコ動画\(RC\)‐'.toutf8
+
+    def initialize mail=nil, password=nil
+      @mail = mail
+      @password = password
+      @logged_in = false
+      @agent = WWW::Mechanize.new()
+      
+      # for parameters current video
+      @video_id = nil
+      @params   = nil
+      @page     = nil
+      self
+    end
+    
+    def login mail=nil, password=nil
+      if !@logged_in
+        @mail = mail unless @mail
+        @password = password unless @password
+        account = {'mail' => @mail, 'password' => @password }
+        
+        begin
+          @page = @agent.post('https://secure.nicovideo.jp/secure/login?site=niconico', account)
+          puts_debug @page.title
+          @logged_in = true
+        rescue
+          @logged_in = false
+          raise NicovideoLoginError
+        end
+      end
+      self
+    end
+
+    def watch(video_id)
+      @video_id = video_id
+      if block_given?
+        yield self
+      end
+    end
+
+    def logged_in?()      @logged_in end
+    def comments(num=500) self.get_comments(@video_id, num) end
+    def flv()             self.get_flv @video_id end
+    def title()           self.get_title @video_id end
+    def tags()            self.get_tags @video_id end
+
+    def get_tags video_id
+      page = get_page(video_id)
+      div = page.parser.search("div#video_tags")
+      tags = div.to_html.scan(/<a href=\"tag\/[\w\%]+?\">(.+?)<\/a>/ou).inject([]) {|arr, v|
+        puts_debug v[0]
+        arr << v[0]
+      }
+      tags
+    end
+    
+    def get_title video_id
+      page = get_page(video_id)
+      title = page.title
+      puts_info title.toutf8.sub(/^#{TITLE_NICONICO}/ou, '')
+      title.toutf8.sub(/^#{TITLE_NICONICO}/ou, '')
+    end
+    
+    def get_video video_id=nil
+      puts_info 'getting video : id = ' + video_id
+      self.get_flv video_id
+    end
+    
+    def get_flv video_id=nil
+      puts_info 'getting flv : id = ' + video_id if video_id
+      begin
+        params = get_params video_id
+        
+        video_url = CGI.unescape(params['url'])
+        video_flv = @agent.get_file(video_url)
+        Video.new(video_id, video_flv)
+      end
+    end
+    
+    def get_comments video_id, num=500
+      puts_info 'getting comment xml : id = ' + video_id
+      begin
+        params = get_params video_id
+        raise unless params['ms']
+        
+        thread_id = params['thread_id']
+        body = %!<thread res_from="-#{num}" version="20061206" thread="#{thread_id}" />!
+        post_url = CGI.unescape(params['ms'])
+        comment_xml = @agent.post_data(post_url, body).body
+        puts_debug comment_xml
+        Comments.new(video_id, comment_xml)
+      end
+    end
+
+    private
+    def get_params video_id=nil
+      self.login unless @logged_in
+      
+      begin
+        if !((video_id == @video_id) && @params)
+          puts_info 'getting params : id = ' + video_id
+          @video_id = video_id
+          page = get_page video_id
+          content = @agent.get_file('http://www.nicovideo.jp/api/getflv?v=' + video_id)
+          puts_debug content
+          @params = content.scan(/([^&]+)=([^&]*)/).inject({}){|h, v| h[v[0]] = v[1]; h}
+        else
+          puts_info 'params are already in cache : id = ' + video_id
+        end
+        @params
+      rescue
+        raise NicovideoNotFoundError
+      end
+    end
+
+    def get_page video_id
+      puts_info 'getting HTML page : id = ' + video_id
+      @agent.get('http://www.nicovideo.jp/watch/' + video_id)
+    end
+
+    def puts_error str ; puts str if (NV_DEBUG_LEVEL >= 1) ; end
+    def puts_info  str ; puts str if (NV_DEBUG_LEVEL >= 2) ; end
+    def puts_debug str ; puts str if (NV_DEBUG_LEVEL >= 3) ; end
+
+  end
+
+  def Nicovideo.new(mail, password)
+    Base.new(mail, password)
+  end
+
+  def Nicovideo.login(mail, password)
+    Base.new(mail, password).login
+  end
+end

data/lib/nicovideo/comments.rb

@@ -0,0 +1,9 @@
+class Nicovideo::Comments
+  def initialize video_id, xml
+    @video_id = video_id
+    @xml = xml
+  end
+
+  def to_s() @xml.to_s end
+  def to_xml() @xml.to_s end
+end

data/lib/nicovideo/mechanize-ext.rb

@@ -0,0 +1,20 @@
+# file: mechanize-ext.rb
+require 'mechanize'
+
+module WWW
+  class Mechanize
+    self.class_eval {
+      def post_data(url, data='', enctype=nil)
+        cur_page = current_page || Page.new( nil, {'content-type'=>'text/html'})
+        request_data = data
+        abs_url = to_absolute_uri(url, cur_page)
+        request = fetch_request(abs_url, :post)
+        request.add_field('Content-Length', request_data.size.to_s)
+        
+        page = fetch_page(abs_url, request, cur_page, [request_data])
+        add_to_history(page)
+        page
+      end
+    }
+  end
+end

data/lib/nicovideo/version.rb

@@ -0,0 +1,9 @@
+module Nicovideo #:nodoc:
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 1
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/lib/nicovideo/video.rb

@@ -0,0 +1,14 @@
+require 'nicovideo/base'
+
+module Nicovideo
+  class Video
+    def initialize video_id, data
+      @video_id = video_id
+      @data     = data
+    end
+    
+    def id()    @video_id end
+    def flv()   @data end
+    def video() @data end
+  end
+end

data/sample/nv_download.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'nicovideo'
+require 'yaml'
+
+video_ids = ARGV
+
+# set account
+account = YAML.load_file(ENV['HOME'] + '/.nicovideo/account.yml')
+mail = account['mail']
+password = account['password']
+
+# you must login to Nicovideo
+nv = Nicovideo.new(mail, password).login
+
+# get videos and comments
+video_ids.each {|video_id|
+
+  nv.watch(video_id) {|v|
+    puts 'title = ' + v.title
+    puts 'tags = ' + v.tags.join(' ')
+    puts 'getting comments xml'
+    File.open("#{video_id}.xml", "wb") {|f| f.write v.comments(100).to_xml }
+    puts 'getting flv file'
+    File.open("#{video_id}.flv", "wb") {|f| f.write v.flv }
+  }
+
+  sleep 3
+}

data/test/test_helper.rb

@@ -0,0 +1,2 @@
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/nicovideo'

data/test/test_nicovideo.rb

@@ -0,0 +1,11 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class TestNicovideo < Test::Unit::TestCase
+
+  def setup
+  end
+  
+  def test_truth
+    assert true
+  end
+end

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.4
+specification_version: 1
+name: nicovideo
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+date: 2008-01-06 00:00:00 +09:00
+summary: utils for nicovideo
+require_paths: 
+- lib
+email: emergent22 (at) livedoor.com
+homepage: http://nicovideo.rubyforge.org
+rubyforge_project: nicovideo
+description: utils for nicovideo
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- emergent
+files: 
+- README.txt
+- Rakefile
+- config/hoe.rb
+- config/requirements.rb
+- lib/nicovideo.rb
+- lib/nicovideo/version.rb
+- lib/nicovideo/mechanize-ext.rb
+- lib/nicovideo/base.rb
+- lib/nicovideo/video.rb
+- lib/nicovideo/comments.rb
+- test/test_helper.rb
+- test/test_nicovideo.rb
+- sample/nv_download.rb
+test_files: 
+- test/test_nicovideo.rb
+- test/test_helper.rb
+rdoc_options: 
+- --main
+- README.txt
+extra_rdoc_files: 
+- README.txt
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: mechanize
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.6.0
+    version: