youtubescraper 0.0.7

data/CHANGELOG

@@ -0,0 +1,20 @@
+0.0.1 2006-11-26
+  First public release
+
+0.0.2 2006-12-03
+  Add rdoc.
+  Add each() method into YouTube::SearchResultScraper
+
+0.0.3 2006-12-22
+  Add error check.
+  Add attribute for video_count, video_from, video_to
+
+0.0.4 2007-02-01
+  Add error check for scraping of pagination.
+  Fix scraping rule for html markup change of youtube.
+
+0.0.5 2007-02-16
+  Fix scraping rule for video_count, video_from, video_to
+
+0.0.6 2007-02-16
+  Fix error handling for video_count, video_from, video_to

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (C) 2006 by in3c.org                                                                               
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,23 @@
+Introduction
+
+  Youtube::SearchResultScraper scrapes video information from search result page on www.youtube.com.
+  You can get result as array or xml.
+  XML format is same as YouTube Developer API (www.youtube.com/dev_api_ref?m=youtube.videos.list_by_tag).
+
+Example
+
+  require "rubygems"
+  require "youtube/searchresultscraper"
+
+  scraper = Youtube::SearchResultScraper.new(keyword, page)
+  scraper.open
+  scraper.scrape
+  puts scraper.get_xml
+
+More Information
+
+  http://www.ark-web.jp/sandbox/wiki/184.html (japanese only)
+
+  Author:	Yuki SHIDA, shida@in3c.org
+  Version:	0.0.2
+  License:	MIT license

data/lib/youtube/browsescraper.rb

@@ -0,0 +1,260 @@
+#--
+# Copyright (C) 2006 by in3c.org, ARK-Web co., ltd
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+# :main:Youtube::BrowseScraper
+# :title:Youtube::BrowseScraper RDoc Documentation
+
+require 'open-uri'
+require 'cgi'
+require 'rubygems'
+require 'hpricot'
+require 'video'
+
+module Youtube #:nodoc:
+  # = Introduction
+  # Youtube::BrowseScraper scrapes video information from search result page
+  # on http://www.youtube.com.
+  #
+  # You can get result as array or xml.
+  #
+  # XML format is same as YouTube Developer API
+  # (http://www.youtube.com/dev_api_ref?m=youtube.videos.list_by_tag).
+  #
+  # = Example
+  #   require "rubygems"
+  #   require "youtube/BrowseScraper"
+  #
+  #   scraper = Youtube::BrowseScraper.new(browse, time, category, language, page)
+  #   scraper.open
+  #   data = scraper.scrape
+  #   p data
+  #
+  # = More Information
+  #
+  #
+  # Author::   Syuichi Kohata <sgkohata@gmail.com>
+  # Version::  0.0.1
+  # License::  MIT license
+
+  class BrowseScraper
+    # constants for browse parameter(default MoseRecent)
+    MostRecent        = 'mr'
+    MostViewed        = 'mp'
+    TopRated          = 'tr'
+    MostDiscussed     = 'md'
+    TopFavorites      = 'mf'
+    MostLinked        = 'mrd'
+    RecentryFeatured  = 'rf'
+    MostResponded     = 'ms'
+    WatchOnMobile     = 'mv'
+    BrowseArray = [MostRecent,
+                   MostViewed,
+                   TopRated,
+                   MostDiscussed,
+                   TopFavorites,
+                   MostLinked,
+                   RecentryFeatured,
+                   MostResponded,
+                   WatchOnMobile]
+
+    # constants for time parameter(default Today)
+    Today       = 't'
+    ThisWeek    = 'w'
+    ThisMonth   = 'm'
+    All         = 'a'
+    TimeArray = [Today,
+                 ThisWeek,
+                 ThisMonth,
+                 All]
+
+    # constants for category parameter(default 0)
+    AllCategory     =  0
+    AutosVehicles   =  2
+    Comedy          = 23
+    Entertainment   = 24
+    FilmAnimation   =  1
+    GadgetsGames    = 20
+    HowtoDIY        = 26
+    Music           = 10
+    NewsPolitics    = 25
+    PeopleBlogs     = 22
+    PetsAnimals     = 15
+    Sports          = 17
+    TravelPlaces    = 19
+
+    # constants for language parameter(default '')
+    AllLanguage   = ''
+    English       = 'EN'
+    Spanish       = 'ES'
+    Japanese      = 'JP'
+    German        = 'DE'
+    Chinese       = 'CN'
+    French        = 'FR'
+    LanguageArray = [AllLanguage,
+                     English,
+                     Spanish,
+                     Japanese,
+                     German,
+                     Chinese,
+                     French]
+
+    attr_accessor :browse
+    attr_accessor :time
+    attr_accessor :category
+    attr_accessor :language
+    attr_accessor :page
+    attr_reader   :video_count
+    attr_reader   :video_from
+    attr_reader   :video_to
+
+    @@youtube_search_base_url = 'http://www.youtube.com/browse'
+
+    # Create Youtube::BrowseScraper object
+    # (default parameter )
+    #
+    # You cannot specify number of videos per page.
+    # Always, the number of videos is 20 per page.
+    def initialize browse = MostRecent, time = Today, category = AllCategory, language = AllLanguage, page = 1
+      @browse   = browse
+      @time     = time
+      @category = category
+      @language = language
+      @page     = page
+
+      errors = []
+      errors << "browse"    if BrowseArray.index(@browse)     == nil
+      errors << "time"      if TimeArray.index(@time)         == nil
+      errors << "language"  if LanguageArray.index(@language) == nil
+      unless errors.empty? then
+        error_msg = "parameter error occurred.\n"
+        errors.each do |error|
+          error_msg << error + " is invalid.\n"
+        end
+        raise error_msg
+      end
+    end
+
+    # Get search result from youtube by specified keyword.
+    def open
+      @url  = @@youtube_search_base_url
+      @url += "?s=#{@browse}"
+      @url += "&t=#{@time}"
+      @url += "&c=#{@category}"
+      @url += "&l=#{@language}"
+      @url += "&p=#{@page}"
+      @html = Kernel.open(@url).read
+      @search_result = Hpricot.parse(@html)
+    end
+
+    # Scrape video information from search result html.
+    def scrape
+      @videos = []
+      @video_count = 0
+      @search_result.search('//div[@class="v120vEntry"]').each do |video_html|
+        video                = Youtube::Video.new
+
+        video.id             = scrape_id(video_html)
+        video.author         = scrape_author(video_html)
+        video.title          = scrape_title(video_html)
+        video.length_seconds = scrape_length_seconds(video_html)
+        video.rating_avg     = scrape_rating_avg(video_html)
+        video.view_count     = scrape_view_count(video_html)
+        video.thumbnail_url  = scrape_thumbnail_url(video_html)
+
+        check_video video
+
+        @videos << video
+        @video_count += 1
+      end
+      @videos
+    end
+
+    # Return videos information as XML Format.
+    def get_xml
+    end
+
+    def replace_document_write_javascript
+      @html.gsub!(%r{<script language="javascript" type="text/javascript">.*?document.write\('(.*?)'\).*?</script>}m, '\1')
+    end
+
+    def scrape_id video_html
+      scrape_thumbnail_url(video_html).sub(%r{.*/([^/]+)/[^/]+.jpg}, '\1')
+    end
+
+    def scrape_thumbnail_url video_html
+      video_html.search("img[@class='vimg120']").to_html.sub(/.*src="(.*?)".*/, '\1')
+    end
+
+    def scrape_title video_html
+      video_html.search('div[@class="vtitle"]/a').inner_html
+    end
+
+    def scrape_length_seconds video_html
+      length_seconds = video_html.search("span[@class='runtime']").inner_html
+      length_seconds =~ /(\d\d):(\d\d)/
+      $1.to_i * 60 + $2.to_i
+    end
+
+    def scrape_rating_avg video_html
+      video_html.search("img[@src='/img/icn_star_full_11x11.gif']").size +
+      video_html.search("img[@src='/img/icn_star_half_11x11.gif']").size * 0.5
+    end
+
+    def scrape_thumbnail_url video_html
+      video_html.search("img[@class=' vimg ']").to_html.sub(/.*src="(.*?)".*/, '\1')
+    end
+
+    def scrape_author video_html
+      video_html.search("div[@class='vfacets']").inner_html.sub(/.*From:<\/span> <a.*?>(.*?)<\/a>.*/m, '\1')
+    end
+
+    def scrape_view_count video_html
+      @num = video_html.search("div[@class='vfacets']").inner_html.sub(/.*Views:<\/span> ([\d,]+).*/m, '\1')
+      @num.gsub(/,/, '').to_i
+    end
+
+    def check_video video
+      errors = []
+
+      errors << "id"             if video.id.empty?
+      errors << "author"         if video.author.empty?
+      errors << "title"          if video.title.empty?
+      errors << "length_seconds" if video.length_seconds.to_s.empty?
+      errors << "thumbnail_url"  if video.thumbnail_url.empty?
+
+      unless errors.empty? then
+        error_msg = "scraping error occurred.\n"
+        errors.each do |error|
+          error_msg << error + " is not setted.\n"
+        end
+        raise error_msg
+      end
+    end
+
+    def each
+      @videos.each do |video|
+        yield video
+      end
+    end
+
+  end
+end

data/lib/youtube/searchresultscraper.rb

@@ -0,0 +1,263 @@
+#--
+# Copyright (C) 2006 by in3c.org, ARK-Web co., ltd
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+# :main:Youtube::SearchResultScraper
+# :title:Youtube::SearchResultScraper RDoc Documentation
+
+require 'open-uri'
+require 'cgi'
+require 'rubygems'
+require 'hpricot'
+require 'youtube/video'
+
+module Youtube #:nodoc:
+
+  # = Introduction
+  # Youtube::SearchResultScraper scrapes video information from search result page
+  # on http://www.youtube.com.
+  #
+  # You can get result as array or xml.
+  #
+  # XML format is same as YouTube Developer API
+  # (http://www.youtube.com/dev_api_ref?m=youtube.videos.list_by_tag).
+  #
+  # = Example
+  #   require "rubygems"
+  #   require "youtube/searchresultscraper"
+  #
+  #   scraper = Youtube::SearchResultScraper.new(keyword, page)
+  #   scraper.open
+  #   scraper.scrape
+  #   puts scraper.get_xml
+  #
+  # = More Information
+  # http://www.ark-web.jp/sandbox/wiki/184.html (japanese only)
+  #
+  # Author::   Yuki SHIDA <shida@in3c.org>
+  # Author::   Konuma Akio <konuma@ark-web.jp>
+  # Version::  0.0.3
+  # License::  MIT license
+
+  class SearchResultScraper
+
+    attr_accessor :keyword
+    attr_accessor :page
+    attr_reader   :video_count
+    attr_reader   :video_from
+    attr_reader   :video_to
+
+    @@youtube_search_base_url = "http://www.youtube.com/results?search_query="
+
+    # Create Youtube::SearchResultScraper object specifying keyword and number of page.
+    #
+    # You cannot specify number of videos per page.
+    # Always, the number of videos is 20 per page.
+    #
+    # * keyword - specify keyword that you want to search on YouTube.
+    #   You must specify keyword encoded by UTF-8.
+    # * page    - specify number of page
+
+    def initialize keyword, page=nil
+      @keyword = keyword
+      @page    = page if not page == nil
+    end
+
+    # Get search result from youtube by specified keyword.
+    def open
+      @url = @@youtube_search_base_url + CGI.escape(@keyword)
+      @url += "&page=#{@page}" if not @page == nil
+      @html = Kernel.open(@url).read
+      replace_document_write_javascript
+      @search_result = Hpricot.parse(@html)
+    end
+
+    # Scrape video information from search result html.
+    def scrape
+      @videos = []
+
+      @search_result.search("//div[@class='vEntry']").each do |video_html|
+        video = Youtube::Video.new
+        video.id             = scrape_id(video_html)
+        video.author         = scrape_author(video_html)
+        video.title          = scrape_title(video_html)
+        video.length_seconds = scrape_length_seconds(video_html)
+        video.rating_avg     = scrape_rating_avg(video_html)
+        video.rating_count   = scrape_rating_count(video_html)
+        video.description    = scrape_description(video_html)
+        video.view_count     = scrape_view_count(video_html)
+        video.thumbnail_url  = scrape_thumbnail_url(video_html)
+        video.tags           = scrape_tags(video_html)
+        video.url            = scrape_url(video_html)
+
+        check_video video
+
+        @videos << video
+      end
+
+      @video_count = scrape_video_count
+      @video_from  = scrape_video_from
+      @video_to    = scrape_video_to
+
+      raise "scraping error" if (is_no_result != @videos.empty?)
+
+      @videos
+    end
+
+    # Iterator for scraped videos.
+    def each
+      @videos.each do |video|
+        yield video
+      end
+    end
+
+    # Return videos information as XML Format.
+    def get_xml
+      xml = "<ut_response status=\"ok\">" +
+              "<video_count>" + @video_count.to_s +  "</video_count>" +
+              "<video_list>\n"
+      each do |video|
+        xml += video.to_xml
+      end
+      xml += "</video_list></ut_response>"
+    end
+
+    private
+
+    def replace_document_write_javascript
+      @html.gsub!(%r{<script language="javascript" type="text/javascript">.*?document.write\('(.*?)'\).*?</script>}m, '\1')
+    end
+
+    def scrape_id video_html
+      scrape_thumbnail_url(video_html).sub(%r{.*/([^/]+)/[^/]+.jpg}, '\1')
+    end
+
+    def scrape_author video_html
+      video_html.search("div[@class='vfacets']").inner_html.sub(/.*From:<\/span> <a.*?>(.*?)<\/a>.*/m, '\1')
+    end
+
+    def scrape_title video_html
+      video_html.search("div[@class='vtitle']/a").inner_html
+    end
+
+    def scrape_length_seconds video_html
+      length_seconds = video_html.search("span[@class='runtime']").inner_html
+      length_seconds =~ /(\d\d):(\d\d)/
+      $1.to_i * 60 + $2.to_i
+    end
+
+    def scrape_rating_avg video_html
+      video_html.search("img[@src='/img/star_sm.gif']").size +
+        video_html.search("img[@src='/img/star_sm_half.gif']").size * 0.5
+    end
+
+    def scrape_rating_count video_html
+      video_html.search("div[@class='rating']").inner_html.sub(/(\d+) rating/, '\1').to_i
+    end
+
+    def scrape_description video_html
+      description = video_html.search("div[@class='vdesc']/span").inner_html.sub(/^\n\t(.*?)\n\t$/m, '\1')
+    end
+
+    def scrape_view_count video_html
+      video_html.search("div[@class='vfacets']").inner_html.sub(/.*Views:<\/span> (\d+).*/m, '\1').to_i
+    end
+
+    def scrape_tags video_html
+      tags = []
+      video_html.search("div[@class='vtagValue']/a").each do |tag|
+        tags << tag.inner_html
+      end
+      tags.join(" ")
+    end
+
+    def scrape_thumbnail_url video_html
+      video_html.search("img[@class='vimg120']").to_html.sub(/.*src="(.*?)".*/, '\1')
+    end
+
+    def scrape_url video_html
+      "http://www.youtube.com" +
+        video_html.search("div[@class='vtitle']/a").to_html.sub(/.*href="(.*?)".*/m, '\1')
+    end
+
+    def scrape_result_header
+      @search_result.search("div[@id='sectionHeader']").inner_html
+    end
+
+    def scrape_video_count
+      video_count = scrape_result_header
+      unless video_count.sub!(/.+Results \d+-\d+ of\s*(|about )([0-9,]+)/m , '\2')
+        raise "no video count: " + @url unless is_no_result
+      end
+      video_count.gsub!(/,/, '')
+      video_count.to_i
+    end
+
+    def scrape_video_from
+      video_from = scrape_result_header
+      unless video_from.sub!(/.+Results (\d+)/m, '\1')
+        raise "no video from: " + @url unless is_no_result
+      end
+      video_from.to_i
+    end
+
+    def scrape_video_to
+      video_to = scrape_result_header
+      unless video_to.sub!(/.+Results \d+-(\d+)/m, '\1')
+        raise "no video to: "  + @url unless is_no_result
+      end
+      video_to.to_i
+    end
+
+    def is_no_result
+      if @is_no_result == nil
+        @is_no_result = @html.include?('No Videos found')
+      end
+      @is_no_result
+    end
+
+    def check_video video
+      errors = []
+
+      errors << "author"         if video.author.empty?
+      errors << "id"             if video.id.empty?
+      errors << "title"          if video.title.empty?
+      errors << "length_seconds" if video.length_seconds.to_s.empty?
+      errors << "rating_avg"     if video.rating_avg.to_s.empty?
+      errors << "rating_count"   if video.rating_count.to_s.empty?
+      errors << "description"    if video.description.empty?
+      errors << "view_count"     if video.view_count.to_s.empty?
+      errors << "tags"           if video.tags.empty?
+      errors << "url"            if video.url.empty?
+      errors << "thumbnail_url"  if video.thumbnail_url.empty?
+
+      unless errors.empty? then
+        error_msg = "scraping error occurred.\n"
+        errors.each do |error|
+          error_msg << error + " is not setted.\n"
+        end
+        raise error_msg
+      end
+    end
+
+  end
+
+end