youtubesearchresultscraper 0.0.3 → 0.0.4

data/CHANGELOG

@@ -5,3 +5,10 @@
   Add rdoc.
   Add each() method into YouTube::SearchResultScraper
 
+0.0.3 2006-12-22
+  Add error check.
+  Add attribute for video_count, video_from, video_to
+
+0.0.4 2007-02-01
+  Add error check for scraping of pagination.
+  Fix scraping rule for html markup change of youtube.

data/lib/youtube/searchresultscraper.rb

@@ -48,7 +48,7 @@ module Youtube #:nodoc:
   #   scraper.open
   #   scraper.scrape
   #   puts scraper.get_xml
-  # 
+  #
   # = More Information
   # http://www.ark-web.jp/sandbox/wiki/184.html (japanese only)
   #
@@ -64,7 +64,7 @@ module Youtube #:nodoc:
     attr_reader   :video_count
     attr_reader   :video_from
     attr_reader   :video_to
-    
+
     @@youtube_search_base_url = "http://www.youtube.com/results?search_query="
 
     # Create Youtube::SearchResultScraper object specifying keyword and number of page.
@@ -123,7 +123,7 @@ module Youtube #:nodoc:
     end
 
     # Iterator for scraped videos.
-    def each 
+    def each
       @videos.each do |video|
         yield video
       end
@@ -131,7 +131,7 @@ module Youtube #:nodoc:
 
     # Return videos information as XML Format.
     def get_xml
-      xml = "<ut_response status=\"ok\">" + 
+      xml = "<ut_response status=\"ok\">" +
               "<video_count>" + @video_count.to_s +  "</video_count>" +
               "<video_list>\n"
       each do |video|
@@ -199,23 +199,29 @@ module Youtube #:nodoc:
     end
 
     def scrape_result_header
-      @search_result.search("div[@id='sectionHeader']/div[@class='my']").inner_html
+      @search_result.search("div[@id='sectionHeader']/div").inner_html
     end
 
     def scrape_video_count
-      scrape_result_header.sub(/.+of *(\d+)/m , '\1').to_i 
+      video_count = scrape_result_header.sub(/.+of *(\d+)/m , '\1')
+      raise "no video count" if video_count.empty?
+      video_count.to_i
     end
 
     def scrape_video_from
-      scrape_result_header.sub(/Results *(\d+)-.+/m, '\1').to_i
+      video_from = scrape_result_header.sub(/Results *(\d+)-.+/m, '\1')
+      raise "no video from" if video_from.empty?
+      video_from.to_i
     end
 
     def scrape_video_to
-      scrape_result_header.sub(/Results.+-(\d+) *of.+/m, '\1').to_i
+      video_to = scrape_result_header.sub(/Results.+-(\d+) *of.+/m, '\1')
+      raise "no video to" if video_to.empty?
+      video_to.to_i
     end
 
     def is_no_result
-        @search_result.search("div[@class='body']").inner_html.include?('No Videos found')
+        @html.include?('No Videos found')
     end
 
     def check_video video

data/lib/youtube/searchresultscraper.rb~

@@ -0,0 +1,250 @@
+#--
+# Copyright (C) 2006 by in3c.org, ARK-Web co., ltd
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+# :main:Youtube::SearchResultScraper
+# :title:Youtube::SearchResultScraper RDoc Documentation
+
+require 'open-uri'
+require 'cgi'
+require 'rubygems'
+require 'hpricot'
+require 'youtube/video'
+
+module Youtube #:nodoc:
+
+  # = Introduction
+  # Youtube::SearchResultScraper scrapes video information from search result page
+  # on http://www.youtube.com.
+  #
+  # You can get result as array or xml.
+  #
+  # XML format is same as YouTube Developer API
+  # (http://www.youtube.com/dev_api_ref?m=youtube.videos.list_by_tag).
+  #
+  # = Example
+  #   require "rubygems"
+  #   require "youtube/searchresultscraper"
+  #
+  #   scraper = Youtube::SearchResultScraper.new(keyword, page)
+  #   scraper.open
+  #   scraper.scrape
+  #   puts scraper.get_xml
+  #
+  # = More Information
+  # http://www.ark-web.jp/sandbox/wiki/184.html (japanese only)
+  #
+  # Author::   Yuki SHIDA <shida@in3c.org>
+  # Author::   Konuma Akio <konuma@ark-web.jp>
+  # Version::  0.0.3
+  # License::  MIT license
+
+  class SearchResultScraper
+
+    attr_accessor :keyword
+    attr_accessor :page
+    attr_reader   :video_count
+    attr_reader   :video_from
+    attr_reader   :video_to
+
+    @@youtube_search_base_url = "http://www.youtube.com/results?search_query="
+
+    # Create Youtube::SearchResultScraper object specifying keyword and number of page.
+    #
+    # You cannot specify number of videos per page.
+    # Always, the number of videos is 20 per page.
+    #
+    # * keyword - specify keyword that you want to search on YouTube.
+    #   You must specify keyword encoded by UTF-8.
+    # * page    - specify number of page
+
+    def initialize keyword, page=nil
+      @keyword = keyword
+      @page    = page if not page == nil
+    end
+
+    # Get search result from youtube by specified keyword.
+    def open
+      url = @@youtube_search_base_url + CGI.escape(@keyword)
+      url += "&page=#{@page}" if not @page == nil
+      @html = Kernel.open(url).read
+      replace_document_write_javascript
+      @search_result = Hpricot.parse(@html)
+    end
+
+    # Scrape video information from search result html.
+    def scrape
+      @videos = []
+
+      @search_result.search("//div[@class='vEntry']").each do |video_html|
+        video = Youtube::Video.new
+        video.id             = scrape_id(video_html)
+        video.author         = scrape_author(video_html)
+        video.title          = scrape_title(video_html)
+        video.length_seconds = scrape_length_seconds(video_html)
+        video.rating_avg     = scrape_rating_avg(video_html)
+        video.rating_count   = scrape_rating_count(video_html)
+        video.description    = scrape_description(video_html)
+        video.view_count     = scrape_view_count(video_html)
+        video.thumbnail_url  = scrape_thumbnail_url(video_html)
+        video.tags           = scrape_tags(video_html)
+        video.url            = scrape_url(video_html)
+
+        check_video video
+
+        @videos << video
+      end
+
+      @video_count = scrape_video_count
+      @video_from  = scrape_video_from
+      @video_to    = scrape_video_to
+      raise "no video count" if @video_count == nil
+      raise "no video from"  if @video_from == nil
+      raise "no video to"    if @video_to == nil
+
+      raise "scraping error" if (is_no_result != @videos.empty?)
+
+      @videos
+    end
+
+    # Iterator for scraped videos.
+    def each
+      @videos.each do |video|
+        yield video
+      end
+    end
+
+    # Return videos information as XML Format.
+    def get_xml
+      xml = "<ut_response status=\"ok\">" +
+              "<video_count>" + @video_count.to_s +  "</video_count>" +
+              "<video_list>\n"
+      each do |video|
+        xml += video.to_xml
+      end
+      xml += "</video_list></ut_response>"
+    end
+
+    private
+
+    def replace_document_write_javascript
+      @html.gsub!(%r{<script language="javascript" type="text/javascript">.*?document.write\('(.*?)'\).*?</script>}m, '\1')
+    end
+
+    def scrape_id video_html
+      scrape_thumbnail_url(video_html).sub(%r{.*/([^/]+)/[^/]+.jpg}, '\1')
+    end
+
+    def scrape_author video_html
+      video_html.search("div[@class='vfacets']").inner_html.sub(/.*From:<\/span> <a.*?>(.*?)<\/a>.*/m, '\1')
+    end
+
+    def scrape_title video_html
+      video_html.search("div[@class='vtitle']/a").inner_html
+    end
+
+    def scrape_length_seconds video_html
+      length_seconds = video_html.search("span[@class='runtime']").inner_html
+      length_seconds =~ /(\d\d):(\d\d)/
+      $1.to_i * 60 + $2.to_i
+    end
+
+    def scrape_rating_avg video_html
+      video_html.search("img[@src='/img/star_sm.gif']").size +
+        video_html.search("img[@src='/img/star_sm_half.gif']").size * 0.5
+    end
+
+    def scrape_rating_count video_html
+      video_html.search("div[@class='rating']").inner_html.sub(/(\d+) rating/, '\1').to_i
+    end
+
+    def scrape_description video_html
+      description = video_html.search("div[@class='vdesc']/span").inner_html.sub(/^\n\t(.*?)\n\t$/m, '\1')
+    end
+
+    def scrape_view_count video_html
+      video_html.search("div[@class='vfacets']").inner_html.sub(/.*Views:<\/span> (\d+).*/m, '\1').to_i
+    end
+
+    def scrape_tags video_html
+      tags = []
+      video_html.search("div[@class='vtagValue']/a").each do |tag|
+        tags << tag.inner_html
+      end
+      tags.join(" ")
+    end
+
+    def scrape_thumbnail_url video_html
+      video_html.search("img[@class='vimg120']").to_html.sub(/.*src="(.*?)".*/, '\1')
+    end
+
+    def scrape_url video_html
+      "http://www.youtube.com" +
+        video_html.search("div[@class='vtitle']/a").to_html.sub(/.*href="(.*?)".*/m, '\1')
+    end
+
+    def scrape_result_header
+      @search_result.search("div[@id='sectionHeader']/div[@class='my']").inner_html
+    end
+
+    def scrape_video_count
+      scrape_result_header.sub(/.+of *(\d+)/m , '\1').to_i
+    end
+
+    def scrape_video_from
+      scrape_result_header.sub(/Results *(\d+)-.+/m, '\1').to_i
+    end
+
+    def scrape_video_to
+      scrape_result_header.sub(/Results.+-(\d+) *of.+/m, '\1').to_i
+    end
+
+    def is_no_result
+        @search_result.search("div[@class='body']").inner_html.include?('No Videos found')
+    end
+
+    def check_video video
+      errors = []
+
+      errors << "author"         if video.author.empty?
+      errors << "id"             if video.id.empty?
+      errors << "title"          if video.title.empty?
+      errors << "length_seconds" if video.length_seconds.to_s.empty?
+      errors << "rating_avg"     if video.rating_avg.to_s.empty?
+      errors << "rating_count"   if video.rating_count.to_s.empty?
+      errors << "description"    if video.description.empty?
+      errors << "view_count"     if video.view_count.to_s.empty?
+      errors << "tags"           if video.tags.empty?
+      errors << "url"            if video.url.empty?
+      errors << "thumbnail_url"  if video.thumbnail_url.empty?
+
+      unless errors.empty? then
+        error_msg = "scraping error occurred.\n"
+        errors.each do |error|
+          error_msg << error + " is not setted.\n"
+        end
+        raise error_msg
+      end
+    end
+
+  end
+
+end

data/test/youtube_scraper_test.rb

@@ -1,13 +1,13 @@
 #!/usr/bin/ruby
 
-require "runit/testcase"
-require "runit/cui/testrunner"
+require 'test/unit'
+
 
 require "rubygems"
 require 'hpricot'
 require "youtube/searchresultscraper"
 
-class SearchResultScraperTest < RUNIT::TestCase
+class SearchResultScraperTest < Test::Unit::TestCase
 
   def test_scrape
 

data/test/youtube_scraper_test.rb~

@@ -0,0 +1,92 @@
+#!/usr/bin/ruby
+
+#require "runit/testcase"
+#require "runit/cui/testrunner"
+require 'test/unit'
+
+
+require "rubygems"
+require 'hpricot'
+require "youtube/searchresultscraper"
+
+class SearchResultScraperTest < Test::Unit::TestCase
+#class SearchResultScraperTest < RUNIT::TestCase
+
+  def test_scrape
+
+    #
+    #�����
+    #
+    #������̤�����
+    open_and_scrape("http://www.youtube.com/results?search_query=", "doraemon", 2)
+    #������̤��ʤ�
+    open_and_scrape("http://www.youtube.com/results?search_query=", "aeudyr jahafudfhadf ahf", 2) 
+
+    #
+    #�۾��
+    #
+    #������̤����뤬��Not Found��������
+    begin 
+      open_local_file_and_scrape("html/dataY_noMsgY.htm")
+      assert_fail("������٤����顼��ȯ�����Ƥ��ʤ�")
+    rescue RuntimeError =>e
+      #puts e
+    end
+    #������̤��ʤ���Not Found��ʤ����
+    begin
+      open_local_file_and_scrape("html/dataN_noMsgN.htm")
+      assert_fail("������٤����顼��ȯ�����Ƥ��ʤ�")
+    rescue RuntimeError
+      #puts e
+    end
+ 
+    #�����ι��ܤ��������Ǥ��Ƥ��ʤ����
+    begin
+      open_local_file_and_scrape("html/scraping_error.html")
+      assert_fail("������٤����顼��ȯ�����Ƥ��ʤ�")
+    rescue RuntimeError => e
+     # puts e
+    end
+  end
+
+  def test_scrape_video_count
+    
+    scraper = open_and_scrape("http://www.youtube.com/results?search_query=", "doraemon", 2)
+    puts scraper.video_count
+    assert( scraper.video_count > 0 )
+    
+    scraper = open_and_scrape("http://www.youtube.com/results?search_query=", "doraemonifdadfa", 2)
+    puts scraper.video_count
+    assert( scraper.video_count == 0 )
+  end
+
+  def open_and_scrape url, keyword=nil, page=nil
+    scraper = MySearchResultScraper.new(url, keyword, page)
+    scraper.open
+    scraper.scrape
+    scraper
+  end
+
+  def open_local_file_and_scrape url
+    scraper = MySearchResultScraper.new(url)
+    scraper.open_local_file
+    scraper.scrape
+  end
+
+end
+
+class MySearchResultScraper  < Youtube::SearchResultScraper
+   @@youtube_search_base_url = "http://www.youtube.com/results?search_query="
+
+  def initialize url, keyword=nil, page=nil
+    @@youtube_search_base_url = url
+    @keyword = keyword
+    @page    = page if not page == nil
+  end
+
+  def open_local_file
+    @html = Kernel.open(@@youtube_search_base_url).read
+    replace_document_write_javascript
+    @search_result = Hpricot.parse(@html)
+  end
+end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: youtubesearchresultscraper
 version: !ruby/object:Gem::Version 
-  version: 0.0.3
-date: 2006-12-22 00:00:00 +09:00
+  version: 0.0.4
+date: 2007-02-01 00:00:00 +09:00
 summary: This gem provide function to scrape html of search result on youtube
 require_paths: 
 - lib
@@ -30,10 +30,12 @@ authors:
 - Yuki SHIDA
 files: 
 - lib/youtube
+- lib/youtube/searchresultscraper.rb~
 - lib/youtube/video.rb
 - lib/youtube/searchresultscraper.rb
 - test/youtube_scraper_test.rb
 - test/html
+- test/youtube_scraper_test.rb~
 - test/html/scraping_error.html
 - test/html/dataY_noMsgY.htm
 - test/html/dataN_noMsgN.htm