valda-video_scraper 1.0.1

data/ChangeLog

@@ -0,0 +1,4 @@
+== 1.0.1 / 2009-01-17
+
+* initial release
+

data/README

@@ -0,0 +1,69 @@
+= WWW::VideoScraper
+
+* http://coderepos.org/share/browser/lang/ruby/video_scraper
+* http://github.com/valda/video_scraper/tree/master
+
+== DESCRIPTION:
+
+Web scraping library for video sharing sites.
+
+== FEATURES/PROBLEMS:
+
+Supported sites
+
+* YouTube
+* NICO NICO DOUGA
+* AmebaVision
+* Dailymotion
+* Veoh
+* YourFileHost
+* RedTube
+* Pornhub
+* Ura Agesage
+* MoroTube
+* Pornotube
+* YouPorn
+
+== SYNOPSIS:
+
+   >> require 'www/video_scraper'
+   >> scraper = WWW::VideoScraper.scrape('http://www.youtube.com/watch?v=OFPnvARUOHI')
+   >> scraper.video_url
+   => "http://www.youtube.com/get_video?video_id=OFPnvARUOHI&t=OEgsToPDskIpQJU48rm4-sS1RtbItouY"
+   >> scraper.thumb_url
+   => "http://i.ytimg.com/vi/OFPnvARUOHI/default.jpg"
+
+== REQUIREMENTS:
+
+* WWW::Mechanize
+* Hpricot
+* CGIAlt (recommend)
+
+== INSTALL:
+
+* sudo gem install valda-video_scraper
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 YAMAGUCHI Seiji <valda at underscore.jp>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,144 @@
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'rake/testtask'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/contrib/rubyforgepublisher'
+require 'rake/contrib/sshpublisher'
+require 'fileutils'
+require 'lib/www/video_scraper'
+include FileUtils
+
+NAME              = "video_scraper"
+AUTHOR            = "YAMAGUCHI Seiji"
+EMAIL             = "valda@underscore.jp"
+DESCRIPTION       = "Web scraping library for video sharing sites."
+RUBYFORGE_PROJECT = "video_scraper"
+HOMEPATH          = "http://github.com/valda/video_scraper"
+BIN_FILES         = %w(  )
+
+VERS              = WWW::VideoScraper::VERSION
+REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil
+CLEAN.include ['**/.*.sw?', '*.gem', '.config']
+RDOC_OPTS = [
+       '--title', "#{NAME} documentation",
+       "--charset", "utf-8",
+       "--opname", "index.html",
+       "--line-numbers",
+       "--main", "README",
+       "--inline-source",
+]
+
+task :default => [:test]
+task :package => [:clean]
+
+Rake::TestTask.new("test") do |t|
+       t.libs   << "test"
+       t.pattern = "test/**/*_test.rb"
+       t.verbose = true
+end
+
+spec = Gem::Specification.new do |s|
+       s.name              = NAME
+       s.version           = VERS
+       s.platform          = Gem::Platform::RUBY
+       s.has_rdoc          = true
+       s.extra_rdoc_files  = ["README", "ChangeLog"]
+       s.rdoc_options     += RDOC_OPTS + ['--exclude', '^(examples|extras)/']
+       s.summary           = DESCRIPTION
+       s.description       = DESCRIPTION
+       s.author            = AUTHOR
+       s.email             = EMAIL
+       s.homepage          = HOMEPATH
+       s.executables       = BIN_FILES
+       s.rubyforge_project = RUBYFORGE_PROJECT
+       s.bindir            = "bin"
+       s.require_path      = "lib"
+       #s.autorequire       = ""
+       s.test_files        = Dir["test/*_test.rb"]
+
+       s.add_dependency('mechanize', '>=0.8.4')
+       #s.required_ruby_version = '>= 1.8.2'
+
+       s.files = %w(README ChangeLog Rakefile) +
+               Dir.glob("{bin,doc,test,lib,templates,generator,extras,website,script}/**/*") +
+               Dir.glob("ext/**/*.{h,c,rb}") +
+               Dir.glob("examples/**/*.rb") +
+               Dir.glob("tools/*.rb") +
+               Dir.glob("rails/*.rb")
+
+       s.extensions = FileList["ext/**/extconf.rb"].to_a
+end
+
+Rake::GemPackageTask.new(spec) do |p|
+       p.need_tar = true
+       p.gem_spec = spec
+end
+
+task :install do
+       name = "#{NAME}-#{VERS}.gem"
+       sh %{rake package}
+       sh %{sudo gem install pkg/#{name}}
+end
+
+task :uninstall => [:clean] do
+       sh %{sudo gem uninstall #{NAME}}
+end
+
+
+Rake::RDocTask.new do |rdoc|
+       rdoc.rdoc_dir = 'html'
+       rdoc.options += RDOC_OPTS
+       rdoc.template = "resh"
+       #rdoc.template = "#{ENV['template']}.rb" if ENV['template']
+       if ENV['DOC_FILES']
+               rdoc.rdoc_files.include(ENV['DOC_FILES'].split(/,\s*/))
+       else
+               rdoc.rdoc_files.include('README', 'ChangeLog')
+               rdoc.rdoc_files.include('lib/**/*.rb')
+               rdoc.rdoc_files.include('ext/**/*.c')
+       end
+end
+
+desc "Publish to RubyForge"
+task :rubyforge => [:rdoc, :package] do
+       require 'rubyforge'
+       Rake::RubyForgePublisher.new(RUBYFORGE_PROJECT, 'yamaguchi').upload
+end
+
+desc 'Package and upload the release to rubyforge.'
+task :release => [:clean, :package] do |t|
+       v = ENV["VERSION"] or abort "Must supply VERSION=x.y.z"
+       abort "Versions don't match #{v} vs #{VERS}" unless v == VERS
+       pkg = "pkg/#{NAME}-#{VERS}"
+
+       require 'rubyforge'
+       rf = RubyForge.new.configure
+       puts "Logging in"
+       rf.login
+
+       c = rf.userconfig
+#      c["release_notes"] = description if description
+#      c["release_changes"] = changes if changes
+       c["preformatted"] = true
+
+       files = [
+               "#{pkg}.tgz",
+               "#{pkg}.gem"
+       ].compact
+
+       puts "Releasing #{NAME} v. #{VERS}"
+       rf.add_release RUBYFORGE_PROJECT, NAME, VERS, *files
+end
+
+desc 'Show information about the gem.'
+task :debug_gem do
+       puts spec.to_ruby
+end
+
+desc 'Update gem spec'
+task :gemspec do
+  open("#{NAME}.gemspec", 'w').write spec.to_ruby
+end

data/lib/www/video_scraper/age_sage.rb

@@ -0,0 +1,34 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require File.expand_path(File.dirname(__FILE__) + '/base')
+
+module WWW
+  module VideoScraper
+    class AgeSage < Base
+      url_regex %r!\Ahttp://adult\.agesage\.jp/contentsPage\.html\?mcd=[[:alnum:]]{16}!
+
+      def initialize(url, opt = nil)
+        super
+        do_query
+      end
+
+      private
+      def do_query
+        @request_url = @page_url.sub('.html', '.xml')
+        @response_body = http_get(@request_url)
+        raise FileNotFound if @response_body.nil? or @response_body.empty?
+        xdoc = Hpricot.XML(@response_body.toutf8)
+        if movie = xdoc.at('/movie')
+          @video_url = movie.at('/movieurl').inner_html
+          @thumb_url = movie.at('/thumbnail').inner_html
+          @title = movie.at('/title').inner_html
+          mcd = @page_url.match(%r|agesage\.jp/contentsPage\.html\?mcd=([[:alnum:]]{16})|)[1]
+          @embed_tag = <<-HTML
+<script type="text/javascript" src="http://adult.agesage.jp/js/past_uraui.js"></script>
+<script type="text/javascript">Purauifla("mcd=#{mcd}", 320, 275);</script>
+        HTML
+        end
+      end
+    end
+  end
+end

data/lib/www/video_scraper/ameba_vision.rb

@@ -0,0 +1,28 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require File.expand_path(File.dirname(__FILE__) + '/base')
+
+module WWW
+  module VideoScraper
+    class AmebaVision < Base
+      url_regex %r!\Ahttp://vision\.ameba\.jp/watch\.do.*?\?movie=(\d+)!
+
+      def initialize(url, opt = nil)
+        super
+        do_query
+      end
+
+      private
+      def do_query
+        id = url_regex_match[1]
+        request_url = "http://vision.ameba.jp/api/get/detailMovie.do?movie=#{id}"
+        xml = http_get(request_url)
+        xdoc = Hpricot.XML(xml.toutf8)
+        @title = xdoc.at('//item/title').inner_html
+        @page_url = xdoc.at('//item/link').inner_html
+        @thumb_url = xdoc.at('//item/imageUrlLarge').inner_html
+        @video_url = @thumb_url.sub('//vi', '//vm').sub('/jpg/', '/flv/').sub('_4.jpg', '.flv')
+      end
+    end
+  end
+end

data/lib/www/video_scraper/base.rb

@@ -0,0 +1,60 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+module WWW
+  module VideoScraper
+    class Base
+      attr_reader :page_url, :video_url, :thumb_url, :embed_tag, :title
+
+      ## class methods
+      class << self
+        def url_regex(regex)
+          @url_regex = regex
+        end
+
+        def valid_url?(url)
+          not (url =~ @url_regex).nil?
+        end
+      end
+
+      def initialize(url, opt = nil)
+        @page_url = url
+        @opt = (opt || {})
+        @url_regex_match = self.class.instance_variable_get(:@url_regex).match(@page_url).freeze
+        raise StandardError, "url is not #{self.class.name} link: #{url}" if @url_regex_match.nil?
+      end
+
+      private
+      def url_regex_match; @url_regex_match; end
+
+      def agent
+        @agent ||= WWW::Mechanize.new do |a|
+          a.user_agent_alias = 'Windows IE 6'
+        end
+      end
+
+      def http_get(url, opt = nil)
+        open_opt = {
+          "User-Agent" => "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)",
+        }.merge( opt || {} )
+        if @opt[:cache]
+          unless @opt[:cache].respond_to?(:get) and @opt[:cache].respond_to?(:set)
+            raise RuntimeError, 'As for cache object what responds to :get and :set is required.'
+          end
+          @opt[:logger].debug 'use cache.'
+          cache_key = "#{url}|#{open_opt}"
+          unless content = @opt[:cache].get(cache_key)
+            content = open(url, open_opt) {|fh| fh.read }
+            @opt[:cache].set(cache_key, content)
+          end
+        else
+          content = open(url, open_opt) {|fh| fh.read }
+        end
+        content
+      rescue OpenURI::HTTPError => e
+        raise TryAgainLater, e.to_s if e.to_s.include?('503')
+        raise e
+      end
+
+    end
+  end
+end

data/lib/www/video_scraper/dailymotion.rb

@@ -0,0 +1,36 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require File.expand_path(File.dirname(__FILE__) + '/base')
+
+module WWW
+  module VideoScraper
+    class Dailymotion < Base
+      url_regex %r!\Ahttp://www\.dailymotion\.com/.*?/video/([\w/-]+)!
+
+      def initialize(url, opt = nil)
+        super
+        do_query
+      end
+
+      private
+      def do_query
+        uri = URI.parse(@page_url)
+        html = http_get(@page_url)
+        doc = Hpricot(html.toutf8)
+        doc.search('//script').each do |elem|
+          if m = elem.inner_html.match(/\.addVariable\("video",\s*"([^"]+)"/i)
+            path = CGI.unescape(m[1]).split(/\|\||@@/).first
+            @video_url = URI.join("#{uri.scheme}://#{uri.host}", path).to_s
+          end
+          if m = elem.inner_html.match(/\.addVariable\("preview",\s+"([^"]+)"/)
+            path = CGI.unescape(m[1]).split(/\|\||@@/).first
+            @thumb_url = URI.join("#{uri.scheme}://#{uri.host}", path).to_s
+          end
+        end
+        @title = doc.at('//h1[@class="nav"]').inner_html rescue nil
+        @embed_tag = CGI.unescapeHTML(doc.at('//textarea[@id="video_player_embed_code_text"]').inner_html) rescue nil
+      end
+    end
+  end
+end
+

data/lib/www/video_scraper/moro_tube.rb

@@ -0,0 +1,37 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require File.expand_path(File.dirname(__FILE__) + '/base')
+
+module WWW
+  module VideoScraper
+    class MoroTube < Base
+      url_regex %r!\Ahttp://www\.morotube\.com/watch\.php\?clip=([[:alnum:]]{8})!
+      attr_reader :author, :duration
+
+      def initialize(url, opt = nil)
+        super
+        do_query
+      end
+
+      private
+      def do_query
+        uri = URI.parse(@page_url)
+        uri.path = '/gen_xml.php'
+        uri.query = "type=o&id=#{url_regex_match[1]}"
+        xml = http_get(uri.to_s)
+        xdoc = Hpricot.XML(xml.toutf8)
+        @title = xdoc.search('/root/video/title').inner_html
+        @video_url = xdoc.search('/root/video/file').inner_html
+        @thumb_url = xdoc.search('/root/video/image').inner_html
+        @author = xdoc.search('/root/video/author').inner_html
+        @duration = xdoc.search('/root/video/duration').inner_html
+
+        html = http_get(@page_url)
+        doc = Hpricot(html)
+        doc.search('//input#inpVdoEmbed') do |elem|
+          @embed_tag = elem.attributes['value']
+        end
+      end
+    end
+  end
+end

data/lib/www/video_scraper/nico_video.rb

@@ -0,0 +1,73 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require File.expand_path(File.dirname(__FILE__) + '/base')
+
+module WWW
+  module VideoScraper
+    class NicoVideo < Base
+      url_regex %r!\Ahttp://www\.nicovideo\.jp/watch/([[:alnum:]]+)!
+
+      def initialize(url, opt = nil)
+        super
+        do_query
+      end
+
+      private
+      def login
+        page = agent.post('https://secure.nicovideo.jp/secure/login?site=niconico',
+                           'mail' => @opt[:nico_video_mail],
+                           'password' => @opt[:nico_video_password])
+        raise RuntimeError, 'login failure' unless page.header['x-niconico-authflag'] == '1'
+      end
+
+      def get_flv(id)
+        request_url = "http://www.nicovideo.jp/api/getflv?v=#{id}"
+        page = agent.get(request_url)
+        q = CGI.parse(page.body)
+        raise FileNotFound unless q['url']
+        @video_url = q['url'].first
+      end
+
+      def get_thumb(id)
+        page = agent.get("http://www.nicovideo.jp/api/getthumbinfo/#{id}")
+        xdoc = Hpricot.XML(page.body.toutf8)
+        xdoc.search('//thumbnail_url') do |elem|
+          @thumb_url = elem.inner_html
+        end
+        xdoc.search('//thumb/title') do |elem|
+          @title = elem.inner_html
+        end
+      end
+
+      def get_embed_tag(id)
+        page = agent.get(@page_url)
+        response_body = page.body
+        doc = Hpricot(response_body)
+        doc.search('//form[@name="form_iframe"] //input[@name="input_iframe"]') do |elem|
+          @embed_tag = elem.attributes['value']
+        end
+      end
+
+      def do_query
+        begin
+          login
+          id = url_regex_match[1]
+          get_flv(id)
+          get_thumb(id)
+          get_embed_tag(id)
+        rescue Timeout::Error => e
+          raise TryAgainLater, e.to_s
+        rescue WWW::Mechanize::ResponseCodeError => e
+          case e.response_code
+          when '404', '403'
+            raise FileNotFound, e.to_s
+          when '502'
+            raise TryAgainLater, e.to_s
+          else
+            raise TryAgainLater, e.to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/www/video_scraper/pornhub.rb

@@ -0,0 +1,29 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require File.expand_path(File.dirname(__FILE__) + '/base')
+
+module WWW
+  module VideoScraper
+    class Pornhub < Base
+      url_regex %r|\Ahttp://www\.pornhub\.com/view_video\.php.*viewkey=[[:alnum:]]{20}|
+
+      def initialize(url, opt = nil)
+        super
+        do_query
+      end
+
+      private
+      def do_query
+        html = http_get(@page_url)
+        raise FileNotFound unless m = html.match(/\.addVariable\("options",\s*"([^"]+)"\);/i)
+        @request_url = URI.decode m[1]
+        @response_body = http_get(@request_url)
+        @video_url = @response_body.match(%r|<flv_url>([^<]+)</flv_url>|).to_a[1]
+        if m = @video_url.match(%r|videos/(\d{3}/\d{3}/\d{3})/\d+.flv|)
+          @thumb_url = "http://p1.pornhub.com/thumbs/#{m[1]}/small.jpg"
+        end
+        @embed_tag = html.match(%r|<textarea[^>]+class="share-flag-embed">(<object type="application/x-shockwave-flash".*?</object>)</textarea>|).to_a[1]
+      end
+    end
+  end
+end

data/lib/www/video_scraper/pornotube.rb

@@ -0,0 +1,42 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require File.expand_path(File.dirname(__FILE__) + '/base')
+
+module WWW
+  module VideoScraper
+    class Pornotube < Base
+      url_regex %r!\Ahttp://(?:www\.)?pornotube\.com/(?:media|channels)\.php\?.*m=(\d+)!
+
+      def initialize(url, opt = nil)
+        super
+        do_query
+      end
+
+      private
+      def login
+        agent.post("http://pornotube.com/index.php",
+                   'verifyAge' => 'true',
+                   'bMonth' => '01',
+                   'bDay' => '01',
+                   'bYear' => '1970',
+                   'submit' => 'View All Content')
+      end
+
+      def do_query
+        id = url_regex_match[1]
+
+        login
+        page = agent.get(@page_url)
+        raise FileNotFound unless embed = page.root.at('//object/embed')
+        src = embed.attributes['src']
+        hash = src.match(/\?v=(.*)$/)[1]
+        page = agent.get("http://pornotube.com/player/player.php?#{hash}")
+        q = CGI::parse(page.body)
+        @video_url = "http://#{q['mediaDomain'][0]}.pornotube.com/#{q['userId'][0]}/#{q['mediaId'][0]}.flv"
+        @thumb_url = "http://photo.pornotube.com/thumbnails/video/#{q['userId'][0]}/#{q['mediaId'][0]}.jpg";
+        @image_url = "http://photo.pornotube.com/thumbnails/video/#{q['userId'][0]}/#{q['mediaId'][0]}_full.jpg";
+        @embed_tag = q['embedCode'][0]
+      end
+    end
+  end
+end

data/lib/www/video_scraper/red_tube.rb

@@ -0,0 +1,62 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require File.expand_path(File.dirname(__FILE__) + '/base')
+
+module WWW
+  module VideoScraper
+    class RedTube < Base
+      url_regex %r|\Ahttp://www\.redtube\.com/(\d{4})|
+
+      def initialize(url, opt = nil)
+        super
+        do_query
+      end
+
+      def embed_tag
+        return @embed_tag if @embed_tag
+        url = "http://www.redtube.com/embed/#{content_id}"
+        response_body = http_get(url)
+        doc = Hpricot(response_body)
+        doc.search('//textarea#cpf') do |elem|
+          @embed_tag = elem.inner_html
+        end
+        @embed_tag
+      end
+
+      private
+      def content_id; url_regex_match[1]; end
+
+      def do_query
+        s = content_id || '0'
+        s = '1' if s.empty?
+        pathnr = s.to_i / 1000
+        s = "%07d" % s.to_i
+        pathnr = "%07d" % pathnr
+        xc = %w!R 1 5 3 4 2 O 7 K 9 H B C D X F G A I J 8 L M Z 6 P Q 0 S T U V W E Y N!
+        qsum = 0
+        s.length.times do |i|
+          qsum += s[i,1].to_i * (i + 1)
+        end
+        s1 = qsum.to_s
+        qsum = 0
+        s1.length.times do |i|
+          qsum += s1[i,1].to_i
+        end
+        qstr = "%02d" % qsum
+        code = ''
+        code += xc[s[3] - 48 + qsum + 3]
+        code += qstr[1,1]
+        code += xc[s[0] - 48 + qsum + 2]
+        code += xc[s[2] - 48 + qsum + 1]
+        code += xc[s[5] - 48 + qsum + 6]
+        code += xc[s[1] - 48 + qsum + 5]
+        code += qstr[0,1]
+        code += xc[s[4] - 48 + qsum + 7]
+        code += xc[s[6] - 48 + qsum + 4]
+        content_video = pathnr + '/' + code + '.flv'
+        @video_url = "http://dl.redtube.com/_videos_t4vn23s9jc5498tgj49icfj4678/#{content_video}"
+        # @thumb_url = "http://thumbs.redtube.com/_thumbs/#{pathnr}/#{s}/#{s}_#{'%03d' % i}.jpg"
+      end
+    end
+  end
+end

data/lib/www/video_scraper/veoh.rb

@@ -0,0 +1,30 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require File.expand_path(File.dirname(__FILE__) + '/base')
+
+module WWW
+  module VideoScraper
+    class Veoh < Base
+      url_regex %r!\Ahttp://www\.veoh\.com/videos/([[:alnum:]]+)!
+
+      def initialize(url, opt = nil)
+        super
+        do_query
+      end
+
+      private
+      def do_query
+        @id = url_regex_match[1]
+        request_url = "http://www.veoh.com/rest/video/#{@id}/details"
+        xml = http_get(request_url)
+        @video_url = xml.match(/fullPreviewHashPath="([^"]+)"/).to_a[1]
+        @title = xml.match(/title="([^"]+)"/).to_a[1]
+        @thumb_url = xml.match(/fullMedResImagePath="([^"]+)"/).to_a[1]
+        html = http_get(@page_url)
+        embed_tag = html.match(/\sid="embed"\s[^>]*value="([^"]+)"/).to_a[1]
+        @embed_tag = CGI.unescapeHTML embed_tag
+      end
+    end
+  end
+end
+