valda-video_scraper 1.0.2 → 1.0.3

data/README

@@ -11,6 +11,7 @@ Web scraping library for video sharing sites.
 
 Supported sites
 
+* AdultSatellites
 * AmebaVision
 * Dailymotion
 * MoroTube

data/lib/www/video_scraper.rb

@@ -15,9 +15,9 @@ end
 
 module WWW
   module VideoScraper
-    VERSION = '1.0.2'
+    VERSION = '1.0.3'
 
-    MODULES_NAME = %w(age_sage ameba_vision dailymotion moro_tube
+    MODULES_NAME = %w(adult_satellites age_sage ameba_vision dailymotion moro_tube
                       nico_video pornhub pornotube red_tube tube8 veoh
                       you_porn you_tube your_file_host)
 
@@ -29,7 +29,6 @@ module WWW
     @@options = {
       :logger => nil,
       :cache => nil,
-      :debug => false,
     }
 
     class << self
@@ -50,20 +49,22 @@ module WWW
         yield @@options
       end
 
+      def find_module(url)
+        @@modules.find { |mod| mod.valid_url?(url) }
+      end
+
       # 与えられた URL を処理できるモジュールを @@modules から検索して実行する
       def scrape(url, opt = nil)
         opt = @@options.merge(opt || {})
         opt[:logger] ||= logger
         raise StandardError, "url param is requred" unless url
 
-        @@modules.each do |scraper|
-          if scraper.valid_url?(url)
-            logger.info "scraper: #{scraper.to_s}"
-            logger.info "url: #{url}"
-            return scraper.new(url, opt)
-          end
+        logger.info "url: #{url}"
+        if mod = find_module(url)
+          logger.info "found module: #{mod.to_s}"
+          return mod.scrape(url, opt)
         end
-        logger.info "unsupport site."
+        logger.info "unsupport url."
         return nil
       rescue TimeoutError, Timeout::Error, Errno::ETIMEDOUT => e
         logger.warn "  Timeout : #{e.to_s}"
@@ -80,8 +81,7 @@ module WWW
       private
       def logger
         return @@options[:logger] if @@options[:logger]
-        require 'logger'
-        @@options[:logger] = Logger.new(STDOUT)
+        @@options[:logger] = NullLogger.new
       end
     end
   end

data/lib/www/video_scraper/adult_satellites.rb

@@ -0,0 +1,27 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require File.expand_path(File.dirname(__FILE__) + '/base')
+
+module WWW
+  module VideoScraper
+    class AdultSatellites < Base
+      url_regex %r!http://(?:www\.)?asa\.tv/movie_detail\.php.*!
+
+      def scrape
+        html = http_get(@page_url)
+        doc = Hpricot(html.toutf8)
+        raise FileNotFound unless flashvars = doc.at('//object //param[@name="FlashVars"]')
+        flashvars = CGI.parse(flashvars.attributes['value'])
+        @video_url = flashvars['videoName'][0]
+        uri = URI.parse(@page_url)
+        if m = @video_url.match(%r!/([[:alnum:]]+/[[:alnum:]]+)\.flv!)
+          @thumb_url = "#{uri.scheme}://#{uri.host}/captured/#{m[1]}_1.jpg"
+        end
+        @title = doc.at('//strong[@class="ptitle"]').inner_html rescue nil
+        if embed = doc.at('//input[@name="embed"]')
+          @embed_tag = CGI.unescapeHTML(embed.attributes['value'])
+        end
+      end
+    end
+  end
+end

data/lib/www/video_scraper/age_sage.rb

@@ -7,13 +7,7 @@ module WWW
     class AgeSage < Base
       url_regex %r!\Ahttp://adult\.agesage\.jp/contentsPage\.html\?mcd=[[:alnum:]]{16}!
 
-      def initialize(url, opt = nil)
-        super
-        do_query
-      end
-
-      private
-      def do_query
+      def scrape
         @request_url = @page_url.sub('.html', '.xml')
         @response_body = http_get(@request_url)
         raise FileNotFound if @response_body.nil? or @response_body.empty?

data/lib/www/video_scraper/ameba_vision.rb

@@ -7,13 +7,7 @@ module WWW
     class AmebaVision < Base
       url_regex %r!\Ahttp://vision\.ameba\.jp/watch\.do.*?\?movie=(\d+)!
 
-      def initialize(url, opt = nil)
-        super
-        do_query
-      end
-
-      private
-      def do_query
+      def scrape
         id = url_regex_match[1]
         request_url = "http://vision.ameba.jp/api/get/detailMovie.do?movie=#{id}"
         xml = http_get(request_url)

data/lib/www/video_scraper/base.rb

@@ -5,6 +5,10 @@ module WWW
     class TryAgainLater < RuntimeError; end
     class FileNotFound < RuntimeError; end
 
+    class NullLogger
+      def method_missing(name, *args); return nil; end
+    end
+
     class Base
       attr_reader :page_url, :video_url, :thumb_url, :embed_tag, :title
 
@@ -17,6 +21,12 @@ module WWW
         def valid_url?(url)
           not (url =~ @url_regex).nil?
         end
+
+        def scrape(url, opt = nil)
+          instance = self.new(url, opt)
+          instance.scrape
+          instance
+        end
       end
 
       def initialize(url, opt = nil)
@@ -26,8 +36,14 @@ module WWW
         raise StandardError, "url is not #{self.class.name} link: #{url}" if @url_regex_match.nil?
       end
 
-      private
-      def url_regex_match; @url_regex_match; end
+      def scrape
+        raise StandardError, 'not implemented yet'
+      end
+
+      protected
+      def url_regex_match
+        @url_regex_match
+      end
 
       def agent
         @agent ||= WWW::Mechanize.new do |a|
@@ -35,6 +51,11 @@ module WWW
         end
       end
 
+      def logger
+        return @opt[:logger] if @opt[:logger]
+        @opt[:logger] = NullLogger.new
+      end
+
       def http_get(url, opt = nil)
         open_opt = {
           "User-Agent" => "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)",
@@ -57,7 +78,6 @@ module WWW
         raise TryAgainLater, e.to_s if e.to_s.include?('503')
         raise e
       end
-
     end
   end
 end

data/lib/www/video_scraper/dailymotion.rb

@@ -7,13 +7,7 @@ module WWW
     class Dailymotion < Base
       url_regex %r!\Ahttp://www\.dailymotion\.com/.*?/video/([\w/-]+)!
 
-      def initialize(url, opt = nil)
-        super
-        do_query
-      end
-
-      private
-      def do_query
+      def scrape
         uri = URI.parse(@page_url)
         html = http_get(@page_url)
         doc = Hpricot(html.toutf8)

data/lib/www/video_scraper/moro_tube.rb

@@ -8,13 +8,7 @@ module WWW
       url_regex %r!\Ahttp://www\.morotube\.com/watch\.php\?clip=([[:alnum:]]{8})!
       attr_reader :author, :duration
 
-      def initialize(url, opt = nil)
-        super
-        do_query
-      end
-
-      private
-      def do_query
+      def scrape
         uri = URI.parse(@page_url)
         uri.path = '/gen_xml.php'
         uri.query = "type=o&id=#{url_regex_match[1]}"

data/lib/www/video_scraper/nico_video.rb

@@ -7,9 +7,25 @@ module WWW
     class NicoVideo < Base
       url_regex %r!\Ahttp://www\.nicovideo\.jp/watch/([[:alnum:]]+)!
 
-      def initialize(url, opt = nil)
-        super
-        do_query
+      def scrape
+        begin
+          login
+          id = url_regex_match[1]
+          get_flv(id)
+          get_thumb(id)
+          get_embed_tag(id)
+        rescue Timeout::Error => e
+          raise TryAgainLater, e.to_s
+        rescue WWW::Mechanize::ResponseCodeError => e
+          case e.response_code
+          when '404', '403'
+            raise FileNotFound, e.to_s
+          when '502'
+            raise TryAgainLater, e.to_s
+          else
+            raise TryAgainLater, e.to_s
+          end
+        end
       end
 
       private
@@ -47,27 +63,6 @@ module WWW
           @embed_tag = elem.attributes['value']
         end
       end
-
-      def do_query
-        begin
-          login
-          id = url_regex_match[1]
-          get_flv(id)
-          get_thumb(id)
-          get_embed_tag(id)
-        rescue Timeout::Error => e
-          raise TryAgainLater, e.to_s
-        rescue WWW::Mechanize::ResponseCodeError => e
-          case e.response_code
-          when '404', '403'
-            raise FileNotFound, e.to_s
-          when '502'
-            raise TryAgainLater, e.to_s
-          else
-            raise TryAgainLater, e.to_s
-          end
-        end
-      end
     end
   end
 end

data/lib/www/video_scraper/pornhub.rb

@@ -7,13 +7,7 @@ module WWW
     class Pornhub < Base
       url_regex %r|\Ahttp://www\.pornhub\.com/view_video\.php.*viewkey=[[:alnum:]]{20}|
 
-      def initialize(url, opt = nil)
-        super
-        do_query
-      end
-
-      private
-      def do_query
+      def scrape
         html = http_get(@page_url)
         raise FileNotFound unless m = html.match(/\.addVariable\("options",\s*"([^"]+)"\);/i)
         @request_url = URI.decode m[1]

data/lib/www/video_scraper/pornotube.rb

@@ -7,22 +7,7 @@ module WWW
     class Pornotube < Base
       url_regex %r!\Ahttp://(?:www\.)?pornotube\.com/(?:media|channels)\.php\?.*m=(\d+)!
 
-      def initialize(url, opt = nil)
-        super
-        do_query
-      end
-
-      private
-      def login
-        agent.post("http://pornotube.com/index.php",
-                   'verifyAge' => 'true',
-                   'bMonth' => '01',
-                   'bDay' => '01',
-                   'bYear' => '1970',
-                   'submit' => 'View All Content')
-      end
-
-      def do_query
+      def scrape
         id = url_regex_match[1]
 
         login
@@ -37,6 +22,16 @@ module WWW
         @image_url = "http://photo.pornotube.com/thumbnails/video/#{q['userId'][0]}/#{q['mediaId'][0]}_full.jpg";
         @embed_tag = q['embedCode'][0]
       end
+
+      private
+      def login
+        agent.post("http://pornotube.com/index.php",
+                   'verifyAge' => 'true',
+                   'bMonth' => '01',
+                   'bDay' => '01',
+                   'bYear' => '1970',
+                   'submit' => 'View All Content')
+      end
     end
   end
 end

data/lib/www/video_scraper/red_tube.rb

@@ -5,33 +5,16 @@ require File.expand_path(File.dirname(__FILE__) + '/base')
 module WWW
   module VideoScraper
     class RedTube < Base
-      url_regex %r|\Ahttp://www\.redtube\.com/(\d{4})|
+      url_regex %r|\Ahttp://www\.redtube\.com/(\d+)|
 
-      def initialize(url, opt = nil)
-        super
-        do_query
-      end
-
-      def embed_tag
-        return @embed_tag if @embed_tag
-        url = "http://www.redtube.com/embed/#{content_id}"
-        response_body = http_get(url)
-        doc = Hpricot(response_body)
-        doc.search('//textarea#cpf') do |elem|
-          @embed_tag = elem.inner_html
-        end
-        @embed_tag
-      end
-
-      private
-      def content_id; url_regex_match[1]; end
-
-      def do_query
+      def scrape
         s = content_id || '0'
         s = '1' if s.empty?
         pathnr = s.to_i / 1000
         s = "%07d" % s.to_i
+        logger.debug s
         pathnr = "%07d" % pathnr
+        logger.debug pathnr
         xc = %w!R 1 5 3 4 2 O 7 K 9 H B C D X F G A I J 8 L M Z 6 P Q 0 S T U V W E Y N!
         qsum = 0
         s.length.times do |i|
@@ -54,9 +37,46 @@ module WWW
         code += xc[s[4] - 48 + qsum + 7]
         code += xc[s[6] - 48 + qsum + 4]
         content_video = pathnr + '/' + code + '.flv'
+        @pathnr = pathnr
+        @s = s
         @video_url = "http://dl.redtube.com/_videos_t4vn23s9jc5498tgj49icfj4678/#{content_video}"
-        # @thumb_url = "http://thumbs.redtube.com/_thumbs/#{pathnr}/#{s}/#{s}_#{'%03d' % i}.jpg"
       end
+
+      def thumb_url
+        return @thumb_url if @thumb_url
+        1.upto(10) do |i|
+          url = "http://thumbs.redtube.com/_thumbs/#{@pathnr}/#{@s}/#{@s}_#{'%03d' % i}.jpg"
+          logger.debug url
+          begin
+            uri = URI.parse(url)
+            Net::HTTP.start(uri.host, uri.port) do |http|
+              response = http.head(uri.request_uri,
+                                   {"User-Agent" => "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)"})
+              logger.debug response.code
+              if 200 == response.code.to_i
+                @thumb_url = url
+                return @thumb_url
+              end
+            end
+          rescue TimeoutError, Timeout::Error, Errno::ETIMEDOUT
+          end
+        end
+        nil
+      end
+
+      def embed_tag
+        return @embed_tag if @embed_tag
+        url = "http://www.redtube.com/embed/#{content_id}"
+        response_body = http_get(url)
+        doc = Hpricot(response_body)
+        doc.search('//textarea#cpf') do |elem|
+          @embed_tag = elem.inner_html
+        end
+        @embed_tag
+      end
+
+      private
+      def content_id; url_regex_match[1]; end
     end
   end
 end

data/lib/www/video_scraper/tube8.rb

@@ -5,15 +5,10 @@ require File.expand_path(File.dirname(__FILE__) + '/base')
 module WWW
   module VideoScraper
     class Tube8 < Base
+      attr_reader :video_url_3gp
       url_regex %r!\Ahttp://www\.tube8\.com/.*/(\d+)(?:/|$)!
 
-      def initialize(url, opt = nil)
-        super
-        do_query
-      end
-
-      private
-      def do_query
+      def scrape
         html = http_get(@page_url)
         doc = Hpricot(html.toutf8)
         raise FileNotFound unless flashvars = doc.at('//object //param[@name="FlashVars"]')
@@ -22,6 +17,14 @@ module WWW
         uri = URI.parse(@page_url)
         @thumb_url = URI.join("#{uri.scheme}://#{uri.host}", flashvars['imageUrl'][0]).to_s
         @title = doc.at('//h1[@class="text"]').inner_html rescue nil
+        doc.search('//a').each do |elem|
+          if href = elem.attributes['href']
+            if href.match(/\.3gp$/)
+              @video_url_3gp = href
+              break
+            end
+          end
+        end
       end
     end
   end

data/lib/www/video_scraper/veoh.rb

@@ -7,13 +7,7 @@ module WWW
     class Veoh < Base
       url_regex %r!\Ahttp://www\.veoh\.com/videos/([[:alnum:]]+)!
 
-      def initialize(url, opt = nil)
-        super
-        do_query
-      end
-
-      private
-      def do_query
+      def scrape
         @id = url_regex_match[1]
         request_url = "http://www.veoh.com/rest/video/#{@id}/details"
         xml = http_get(request_url)
@@ -21,10 +15,10 @@ module WWW
         @title = xml.match(/title="([^"]+)"/).to_a[1]
         @thumb_url = xml.match(/fullMedResImagePath="([^"]+)"/).to_a[1]
         html = http_get(@page_url)
-        embed_tag = html.match(/\sid="embed"\s[^>]*value="([^"]+)"/).to_a[1]
-        @embed_tag = CGI.unescapeHTML embed_tag
+        if embed_tag = html.match(/\sid="embed"\s[^>]*value="([^"]+)"/).to_a[1]
+          @embed_tag = CGI.unescapeHTML(embed_tag)
+        end
       end
     end
   end
 end
-

data/lib/www/video_scraper/you_porn.rb

@@ -7,13 +7,7 @@ module WWW
     class YouPorn < Base
       url_regex %r!\Ahttp://youporn\.com/watch/(\d+)!
 
-      def initialize(url, opt = nil)
-        super
-        do_query
-      end
-
-      private
-      def do_query
+      def scrape
         id = url_regex_match[1]
 
         request_url = @page_url.sub(/(\?.*)?$/, '?user_choice=Enter')

data/lib/www/video_scraper/you_tube.rb

@@ -7,16 +7,28 @@ module WWW
     class YouTube < Base
       url_regex %r!\Ahttp://(?:www|jp)\.youtube\.com/watch.*[?&]v=([[:alnum:]]+)!
 
-      def initialize(url, opt = nil)
-        super
-        do_query
+      def scrape
+        page = pass_verify_age
+        @title = page.root.at('//head/title').inner_html.sub(/^YouTube[\s-]*/, '') rescue ''
+        @embed_tag = page.root.at('//input[@id="embed_code"]').attributes['value'] rescue nil
+        page.root.search('//script').each do |script|
+          if m = script.inner_html.match(/var\s+swfArgs\s*=\s*([^;]+);/)
+            swf_args = JSON::parse(m[1])
+            uri = URI.parse(@page_url)
+            uri.path = '/get_video'
+            uri.query = "video_id=#{swf_args['video_id']}&t=#{swf_args['t']}"
+            @video_url = uri.to_s
+            @thumb_url = "http://i.ytimg.com/vi/#{swf_args['video_id']}/default.jpg"
+          end
+        end
+        raise FileNotFound, 'file not found' if @video_url.nil?
       end
 
       private
       def login
         uri = URI.parse(@page_url)
         page = agent.get("#{uri.scheme}://#{uri.host}/login")
-        login_form = page.forms.with.name('loginForm').first
+        login_form = page.form('loginForm')
         login_form.username = @opt[:you_tube_username]
         login_form.password = @opt[:you_tube_password]
         agent.submit(login_form)
@@ -33,23 +45,6 @@ module WWW
         end
         page
       end
-
-      def do_query
-        page = pass_verify_age
-        @title = page.root.at('//head/title').inner_html.sub(/^YouTube[\s-]*/, '') rescue ''
-        @embed_tag = page.root.at('//input[@id="embed_code"]').attributes['value'] rescue nil
-        page.root.search('//script').each do |script|
-          if m = script.inner_html.match(/var\s+swfArgs\s*=\s*([^;]+);/)
-            swf_args = JSON::parse(m[1])
-            uri = URI.parse(@page_url)
-            uri.path = '/get_video'
-            uri.query = "video_id=#{swf_args['video_id']}&t=#{swf_args['t']}"
-            @video_url = uri.to_s
-            @thumb_url = "http://i.ytimg.com/vi/#{swf_args['video_id']}/default.jpg"
-          end
-        end
-        raise FileNotFound, 'file not found' if @video_url.nil?
-      end
     end
   end
 end

data/lib/www/video_scraper/your_file_host.rb

@@ -11,11 +11,6 @@ module WWW
       class BandwidthAllowanceExceeded < TryAgainLater; end
       class NoFileCategory < FileNotFound; end
 
-      def initialize(url, opt = nil)
-        super
-        do_query
-      end
-
       def filename
         uri = URI.parse(@page_url)
         q = CGI.parse(uri.query)
@@ -23,8 +18,7 @@ module WWW
       end
       alias :title :filename
 
-      private
-      def do_query
+      def scrape
         html = http_get(@page_url)
         doc = Hpricot(html.toutf8)
         if elem = doc.at('//object[@id="objectPlayer"] //param[@name="movie"]')

data/test/test_helper.rb

@@ -5,4 +5,19 @@ require File.expand_path(File.dirname(__FILE__) + '/../lib/www/video_scraper')
 
 require 'fileutils'
 require 'filecache'
+require 'logger'
 require 'pit'
+
+class Test::Unit::TestCase
+  def logger
+    @logger ||= Logger.new(STDOUT)
+  end
+
+  def filecache
+    @filecache ||= FileCache.new('TestVideoScraper', '/tmp/test_video_scraper_cache', 60 * 60)
+  end
+
+  def default_opt
+    @default_opt ||= { :logger => logger, :cache => filecache }
+  end
+end

data/test/www/test_video_scraper.rb

@@ -12,9 +12,30 @@ class TestVideoScraper < Test::Unit::TestCase
   end
 
   def test_configure
+    WWW::VideoScraper.options = {}
+
+    assert_nil WWW::VideoScraper.options[:cache]
+    assert_nil WWW::VideoScraper.options[:logger]
+
     WWW::VideoScraper.configure do |conf|
       conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
+      conf[:logger] = Logger.new(STDOUT)
     end
+
     assert_kind_of FileCache, WWW::VideoScraper.options[:cache]
+    assert_kind_of Logger, WWW::VideoScraper.options[:logger]
+  end
+
+  def test_find_module
+    mod = WWW::VideoScraper.find_module('http://jp.youtube.com/watch?v=Ym20IwIUbuU')
+    assert_equal WWW::VideoScraper::YouTube, mod
+    mod = WWW::VideoScraper.find_module('http://www.nicovideo.jp/watch/sm1175788')
+    assert_equal WWW::VideoScraper::NicoVideo, mod
+  end
+
+  def test_scrape
+    vs = WWW::VideoScraper.scrape('http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv')
+    assert_kind_of WWW::VideoScraper::Yourfilehost, vs
   end
 end
+

data/test/www/video_scraper/test_adult_satellites.rb

@@ -0,0 +1,13 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require File.dirname(__FILE__) + '/../../test_helper'
+
+class TestAdultSatellites < Test::Unit::TestCase
+  def test_scrape
+    vs = WWW::VideoScraper::AdultSatellites.scrape('http://www.asa.tv/movie_detail.php?userid=&movie_id=15680', default_opt)
+    assert_equal 'http://www.asa.tv/movie_detail.php?userid=&movie_id=15680', vs.page_url
+    assert_match %r|http://asa\.tv/movie/D5/gcuppapa/[[:alnum:]]{32}\.flv|, vs.video_url
+    assert_match %r|http://www\.asa\.tv/captured/gcuppapa/[[:alnum:]]{32}_1\.jpg|, vs.thumb_url
+    assert_equal '妃乃ひかり,2', vs.title
+  end
+end

data/test/www/video_scraper/test_age_sage.rb

@@ -3,19 +3,8 @@
 require File.dirname(__FILE__) + '/../../test_helper'
 
 class TestAgeSage < Test::Unit::TestCase
-  def setup
-    @cache_root = '/tmp/test_video_scraper_cache'
-    WWW::VideoScraper.configure do |conf|
-      conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
-    end
-  end
-
-  def teardown
-    # FileUtils.remove_entry_secure(@cache_root, true)
-  end
-
   def test_scrape
-    vs = WWW::VideoScraper.scrape('http://adult.agesage.jp/contentsPage.html?mcd=oadyChZgcoN9rqbJ')
+    vs = WWW::VideoScraper::AgeSage.scrape('http://adult.agesage.jp/contentsPage.html?mcd=oadyChZgcoN9rqbJ', default_opt)
     assert_equal 'http://adult.agesage.jp/contentsPage.html?mcd=oadyChZgcoN9rqbJ', vs.page_url
     assert_match %r!http://file\d+\.agesage\.jp/flv/\d{8}/\d{10}_\d{6}\.flv!, vs.video_url
     assert_match %r!http://file\d+\.agesage\.jp/data/\d{8}/\d{10}_\d{6}!, vs.thumb_url

data/test/www/video_scraper/test_ameba_vision.rb

@@ -3,19 +3,8 @@
 require File.dirname(__FILE__) + '/../../test_helper'
 
 class TestAmebaVision < Test::Unit::TestCase
-  def setup
-    @cache_root = '/tmp/test_video_scraper_cache'
-    WWW::VideoScraper.configure do |conf|
-      conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
-    end
-  end
-
-  def teardown
-    # FileUtils.remove_entry_secure(@cache_root, true)
-  end
-
   def test_scrape
-    vs = WWW::VideoScraper.scrape('http://vision.ameba.jp/watch.do?movie=772341')
+    vs = WWW::VideoScraper::AmebaVision.scrape('http://vision.ameba.jp/watch.do?movie=772341', default_opt)
     assert_equal 'http://vision.ameba.jp/watch.do?movie=772341', vs.page_url
     assert_match %r|http://[-[:alnum:]]+\.vision\.ameba\.jp/flv/\d{4}/\d{2}/\d{2}/[[:alnum:]]+\.flv|, vs.video_url
     assert_match %r|http://[-[:alnum:]]+\.vision\.ameba\.jp/jpg/\d{4}/\d{2}/\d{2}/[[:alnum:]]+_4\.jpg|, vs.thumb_url

data/test/www/video_scraper/test_base.rb

@@ -0,0 +1,14 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require File.dirname(__FILE__) + '/../../test_helper'
+
+class TestBase < Test::Unit::TestCase
+  def test_nulllogger
+    logger = WWW::VideoScraper::NullLogger.new
+    assert_nil logger.fatal 'nop'
+    assert_nil logger.error 'nop'
+    assert_nil logger.warn 'nop'
+    assert_nil logger.info 'nop'
+    assert_nil logger.debug 'nop'
+  end
+end

data/test/www/video_scraper/test_dailymotion.rb

@@ -3,19 +3,8 @@
 require File.dirname(__FILE__) + '/../../test_helper'
 
 class TestDailymotion < Test::Unit::TestCase
-  def setup
-    @cache_root = '/tmp/test_video_scraper_cache'
-    WWW::VideoScraper.configure do |conf|
-      conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
-    end
-  end
-
-  def teardown
-    # FileUtils.remove_entry_secure(@cache_root, true)
-  end
-
   def test_scrape
-    vs = WWW::VideoScraper.scrape('http://www.dailymotion.com/mokelov/japan/video/x12gr3_music')
+    vs = WWW::VideoScraper::Dailymotion.scrape('http://www.dailymotion.com/mokelov/japan/video/x12gr3_music', default_opt)
     assert_equal 'http://www.dailymotion.com/mokelov/japan/video/x12gr3_music', vs.page_url
     assert_match %r|http://www\.dailymotion\.com/get/\d{2}/\d+x\d+/flv/\d+\.flv\?key=[[:alnum:]]+|, vs.video_url
     assert_match %r|http://\w+\.static\.dailymotion\.com/dyn/preview/\d+x\d+/\d+\.jpg\?\d{14}|, vs.thumb_url

data/test/www/video_scraper/test_moro_tube.rb

@@ -3,19 +3,8 @@
 require File.dirname(__FILE__) + '/../../test_helper'
 
 class TestMoroTube < Test::Unit::TestCase
-  def setup
-    @cache_root = '/tmp/test_video_scraper_cache'
-    WWW::VideoScraper.configure do |conf|
-      conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
-    end
-  end
-
-  def teardown
-    # FileUtils.remove_entry_secure(@cache_root, true)
-  end
-
   def test_scrape
-    vs = WWW::VideoScraper.scrape('http://www.morotube.com/watch.php?clip=46430e1d')
+    vs = WWW::VideoScraper::MoroTube.scrape('http://www.morotube.com/watch.php?clip=46430e1d', default_opt)
     assert_equal 'http://www.morotube.com/watch.php?clip=46430e1d', vs.page_url
     assert_match %r!http://video\d+\.morotube\.com/[[:alnum:]]{32}/[[:alnum:]]{8}/[[:alnum:]]{8}\.flv!, vs.video_url
     assert_match %r!http://static\d+\.morotube\.com/thumbs/\w{8}\.jpg!, vs.thumb_url

data/test/www/video_scraper/test_nico_video.rb

@@ -4,23 +4,16 @@ require File.dirname(__FILE__) + '/../../test_helper'
 
 class TestNicoVideo < Test::Unit::TestCase
   def setup
-    @cache_root = '/tmp/test_video_scraper_cache'
-    WWW::VideoScraper.configure do |conf|
-      conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
-    end
     config = Pit.get('nicovideo.jp', :require => {
                        'mail' => 'your email in nicovideo.jp',
                        'password' => 'your password in nicovideo.jp'
                      })
-    @opt = { :nico_video_mail => config['mail'], :nico_video_password => config['password'] }
-  end
-
-  def teardown
-    # FileUtils.remove_entry_secure(@cache_root, true)
+    default_opt.merge!(:nico_video_mail => config['mail'],
+                       :nico_video_password => config['password'])
   end
 
   def test_scrape
-    vs = WWW::VideoScraper.scrape('http://www.nicovideo.jp/watch/sm1175788', @opt)
+    vs = WWW::VideoScraper::NicoVideo.scrape('http://www.nicovideo.jp/watch/sm1175788', default_opt)
     assert_equal 'http://www.nicovideo.jp/watch/sm1175788', vs.page_url
     assert_match %r|http://smile-[[:alnum:]]+\.nicovideo\.jp/smile\?v=\d{7}\.\d{5}|, vs.video_url
     assert_match %r|http://tn-skr\d+\.smilevideo\.jp/smile\?i=\d{7}|, vs.thumb_url

data/test/www/video_scraper/test_pornhub.rb

@@ -3,19 +3,8 @@
 require File.dirname(__FILE__) + '/../../test_helper'
 
 class TestPornhub < Test::Unit::TestCase
-  def setup
-    @cache_root = '/tmp/test_video_scraper_cache'
-    WWW::VideoScraper.configure do |conf|
-      conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
-    end
-  end
-
-  def teardown
-    # FileUtils.remove_entry_secure(@cache_root, true)
-  end
-
   def test_scrape
-    vs = WWW::VideoScraper.scrape('http://www.pornhub.com/view_video.php?viewkey=27f115e7fee8c18f92b0')
+    vs = WWW::VideoScraper::Pornhub.scrape('http://www.pornhub.com/view_video.php?viewkey=27f115e7fee8c18f92b0', default_opt)
     assert_equal 'http://www.pornhub.com/view_video.php?viewkey=27f115e7fee8c18f92b0', vs.page_url
     assert_match %r|http://media1.pornhub.com/dl/[[:alnum:]]{32}/[[:alnum:]]{8}/videos/000/191/743/191743\.flv|, vs.video_url
     assert_equal 'http://p1.pornhub.com/thumbs/000/191/743/small.jpg', vs.thumb_url

data/test/www/video_scraper/test_pornotube.rb

@@ -3,19 +3,8 @@
 require File.dirname(__FILE__) + '/../../test_helper'
 
 class TestPornotube < Test::Unit::TestCase
-  def setup
-    @cache_root = '/tmp/test_video_scraper_cache'
-    WWW::VideoScraper.configure do |conf|
-      conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
-    end
-  end
-
-  def teardown
-    # FileUtils.remove_entry_secure(@cache_root, true)
-  end
-
   def test_scrape
-    vs = WWW::VideoScraper.scrape('http://pornotube.com/media.php?m=1677937')
+    vs = WWW::VideoScraper::Pornotube.scrape('http://pornotube.com/media.php?m=1677937', default_opt)
     assert_equal 'http://pornotube.com/media.php?m=1677937', vs.page_url
     assert_match %r|http://video\d+\.pornotube\.com/\d+/\d+\.flv|, vs.video_url
     assert_match %r|http://photo\.pornotube\.com/thumbnails/video/\d+/\d+\.jpg|, vs.thumb_url
@@ -23,7 +12,7 @@ class TestPornotube < Test::Unit::TestCase
   end
 
   def test_scrape_alt_url
-    vs = WWW::VideoScraper.scrape('http://pornotube.com/channels.php?channelId=83&m=1677912')
+    vs = WWW::VideoScraper::Pornotube.scrape('http://pornotube.com/channels.php?channelId=83&m=1677912', default_opt)
     assert_equal 'http://pornotube.com/channels.php?channelId=83&m=1677912', vs.page_url
     assert_match %r|http://video\d+\.pornotube\.com/\d+/\d+\.flv|, vs.video_url
     assert_match %r|http://photo\.pornotube\.com/thumbnails/video/\d+/\d+\.jpg|, vs.thumb_url

data/test/www/video_scraper/test_red_tube.rb

@@ -3,22 +3,11 @@
 require File.dirname(__FILE__) + '/../../test_helper'
 
 class TestRedTube < Test::Unit::TestCase
-  def setup
-    @cache_root = '/tmp/test_video_scraper_cache'
-    WWW::VideoScraper.configure do |conf|
-      conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
-    end
-  end
-
-  def teardown
-    # FileUtils.remove_entry_secure(@cache_root, true)
-  end
-
   def test_scrape
-    vs = WWW::VideoScraper.scrape('http://www.redtube.com/8415')
-    assert_equal 'http://www.redtube.com/8415', vs.page_url
-    assert_match %r!http://dl\.redtube\.com/_videos_t4vn23s9jc5498tgj49icfj4678/0000008/Z2XDJA1ZL\.flv!, vs.video_url
-    assert_nil vs.thumb_url
+    vs = WWW::VideoScraper::RedTube.scrape('http://www.redtube.com/15021', default_opt)
+    assert_equal 'http://www.redtube.com/15021', vs.page_url
+    assert_equal 'http://dl.redtube.com/_videos_t4vn23s9jc5498tgj49icfj4678/0000015/X6KKXB0DB.flv', vs.video_url
+    assert_equal 'http://thumbs.redtube.com/_thumbs/0000015/0015021/0015021_001.jpg', vs.thumb_url
     assert_match %r!<object\s+.*</object>!, vs.embed_tag
   end
 end

data/test/www/video_scraper/test_tube8.rb

@@ -3,22 +3,12 @@
 require File.dirname(__FILE__) + '/../../test_helper'
 
 class TestTube8 < Test::Unit::TestCase
-  def setup
-    @cache_root = '/tmp/test_video_scraper_cache'
-    WWW::VideoScraper.configure do |conf|
-      conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
-    end
-  end
-
-  def teardown
-    # FileUtils.remove_entry_secure(@cache_root, true)
-  end
-
   def test_scrape
-    vs = WWW::VideoScraper.scrape('http://www.tube8.com/anal/alexis-amore-pov/56983/')
+    vs = WWW::VideoScraper::Tube8.scrape('http://www.tube8.com/anal/alexis-amore-pov/56983/', default_opt)
     assert_equal 'http://www.tube8.com/anal/alexis-amore-pov/56983/', vs.page_url
-    assert_match %r|http://medianl\d+\.tube8\.com/flv/[[:alnum:]]{32}/\d{8}/\d{4}/\d{2}/[[:alnum:]]+/[[:alnum:]]+\.flv|, vs.video_url
+    assert_match %r|http://media\w+\.tube8\.com/flv/[[:alnum:]]{32}/[[:alnum:]]{8}/\d{4}/\d{2}/[[:alnum:]]+/[[:alnum:]]+\.flv|, vs.video_url
     assert_equal 'http://www.tube8.com/vs/83/56983.jpg', vs.thumb_url
     assert_equal 'Alexis Amore POV', vs.title
+    assert_match %r!http://mobile\d+\.tube8\.com/flv/[[:alnum:]]{32}/[[:alnum:]]{8}/\d{4}/\d{2}/[[:alnum:]]+/[[:alnum:]]+\.3gp!, vs.video_url_3gp
   end
 end

data/test/www/video_scraper/test_veoh.rb

@@ -3,19 +3,8 @@
 require File.dirname(__FILE__) + '/../../test_helper'
 
 class TestVeoh < Test::Unit::TestCase
-  def setup
-    @cache_root = '/tmp/test_video_scraper_cache'
-    WWW::VideoScraper.configure do |conf|
-      conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
-    end
-  end
-
-  def teardown
-    # FileUtils.remove_entry_secure(@cache_root, true)
-  end
-
   def test_scrape
-    vs = WWW::VideoScraper.scrape('http://www.veoh.com/videos/v6245232rh8aGEM9')
+    vs = WWW::VideoScraper::Veoh.scrape('http://www.veoh.com/videos/v6245232rh8aGEM9', default_opt)
     assert_equal 'http://www.veoh.com/videos/v6245232rh8aGEM9', vs.page_url
     assert_match %r|http://content\.veoh\.com/flash/p/\d/[[:alnum:]]{16}/[[:alnum:]]{40}\.fll\?ct=[[:alnum:]]{48}|, vs.video_url
     assert_match %r|http://p-images\.veoh\.com/image\.out\?imageId=media-[[:alnum:]]+.jpg|, vs.thumb_url

data/test/www/video_scraper/test_you_porn.rb

@@ -3,19 +3,8 @@
 require File.dirname(__FILE__) + '/../../test_helper'
 
 class TestYouPorn < Test::Unit::TestCase
-  def setup
-    @cache_root = '/tmp/test_video_scraper_cache'
-    WWW::VideoScraper.configure do |conf|
-      conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
-    end
-  end
-
-  def teardown
-    # FileUtils.remove_entry_secure(@cache_root, true)
-  end
-
   def test_scrape
-    vs = WWW::VideoScraper.scrape('http://youporn.com/watch/93495?user_choice=Enter')
+    vs = WWW::VideoScraper::YouPorn.scrape('http://youporn.com/watch/93495?user_choice=Enter', default_opt)
     assert_equal 'http://youporn.com/watch/93495?user_choice=Enter', vs.page_url
     assert_match %r|http://download\.youporn\.com/download/\d+/flv/\d+_.*\.flv.*|, vs.video_url
     assert_match %r|http://ss-\d+\.youporn\.com/screenshot/\d+/\d+/screenshot/\d+_large\.jpg|, vs.thumb_url

data/test/www/video_scraper/test_you_tube.rb

@@ -4,24 +4,16 @@ require File.dirname(__FILE__) + '/../../test_helper'
 
 class TestYouTube < Test::Unit::TestCase
   def setup
-    @cache_root = '/tmp/test_video_scraper_cache'
-    WWW::VideoScraper.configure do |conf|
-      conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
-    end
-
     config = Pit.get('youtube.com', :require => {
                        'username' => 'your email in youtube.com',
                        'password' => 'your password in youtube.com'
                      })
-    @opt = { :you_tube_username => config['username'], :you_tube_password => config['password'] }
-  end
-
-  def teardown
-    # FileUtils.remove_entry_secure(@cache_root, true)
+    default_opt.merge!(:you_tube_username => config['username'],
+                       :you_tube_password => config['password'])
   end
 
   def test_scrape
-    vs = WWW::VideoScraper.scrape('http://jp.youtube.com/watch?v=Ym20IwIUbuU', @opt)
+    vs = WWW::VideoScraper::YouTube.scrape('http://jp.youtube.com/watch?v=Ym20IwIUbuU', default_opt)
     assert_equal 'http://jp.youtube.com/watch?v=Ym20IwIUbuU', vs.page_url
     assert_match %r|http://jp\.youtube\.com/get_video\?video_id=Ym20IwIUbuU&t=[-_[:alnum:]]{32}|, vs.video_url
     assert_match %r|http://\w\.ytimg\.com/vi/Ym20IwIUbuU/default\.jpg|, vs.thumb_url
@@ -30,11 +22,11 @@ class TestYouTube < Test::Unit::TestCase
   end
 
   def test_scrape_alt_url
-    vs = WWW::VideoScraper.scrape('http://jp.youtube.com/watch?v=ibhaQZB9TWU', @opt)
-    assert_equal 'http://jp.youtube.com/watch?v=ibhaQZB9TWU', vs.page_url
-    assert_match %r|http://jp\.youtube\.com/get_video\?video_id=ibhaQZB9TWU&t=[-_[:alnum:]]{32}|, vs.video_url
-    assert_match %r|http://\w\.ytimg\.com/vi/ibhaQZB9TWU/default\.jpg|, vs.thumb_url
+    vs = WWW::VideoScraper::YouTube.scrape('http://jp.youtube.com/watch?v=ATHC8qDkoO0', default_opt)
+    assert_equal 'http://jp.youtube.com/watch?v=ATHC8qDkoO0', vs.page_url
+    assert_match %r|http://jp\.youtube\.com/get_video\?video_id=ATHC8qDkoO0&t=[-_[:alnum:]]{32}|, vs.video_url
+    assert_match %r|http://\w\.ytimg\.com/vi/ATHC8qDkoO0/default\.jpg|, vs.thumb_url
     assert_match %r|^<object\s+.*</object>$|, vs.embed_tag
-    assert_equal '水着の人妻', vs.title
+    assert_equal "019' Sexii_Shower 藤澤まお_洗白白", vs.title
   end
 end

data/test/www/video_scraper/test_your_file_host.rb

@@ -3,22 +3,12 @@
 require File.dirname(__FILE__) + '/../../test_helper'
 
 class TestYourFileHost < Test::Unit::TestCase
-  def setup
-    @cache_root = '/tmp/test_video_scraper_cache'
-    WWW::VideoScraper.configure do |conf|
-      conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
-    end
-  end
-
-  def teardown
-    # FileUtils.remove_entry_secure(@cache_root, true)
-  end
-
   def test_scrape
-    vs = WWW::VideoScraper.scrape('http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv')
+    vs = WWW::VideoScraper::YourFileHost.scrape('http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv', default_opt)
     assert_equal 'http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv', vs.page_url
     assert_match %r|http://cdn\w*\.yourfilehost\.com/unit1/flash8/\d+/[[:alnum:]]{32}\.flv|, vs.video_url
     assert_match %r|http://cdn\w*\.yourfilehost\.com/thumbs/\d+/[[:alnum:]]{32}\.jpg|, vs.thumb_url
     assert_equal 'XV436__03.wmv', vs.title
   end
 end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: valda-video_scraper
 version: !ruby/object:Gem::Version 
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors: 
 - YAMAGUCHI Seiji
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-01-18 00:00:00 -08:00
+date: 2009-01-27 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -50,6 +50,7 @@ files:
 - test/www/video_scraper/test_dailymotion.rb
 - test/www/video_scraper/test_age_sage.rb
 - test/www/video_scraper/test_pornotube.rb
+- test/www/video_scraper/test_your_file_host_flymake.rb
 - test/www/video_scraper/test_tube8.rb
 - test/www/video_scraper/test_your_file_host.rb
 - test/www/video_scraper/test_moro_tube.rb
@@ -58,7 +59,9 @@ files:
 - test/www/video_scraper/test_nico_video.rb
 - test/www/video_scraper/test_you_porn.rb
 - test/www/video_scraper/test_you_tube.rb
+- test/www/video_scraper/test_adult_satellites.rb
 - test/www/video_scraper/test_red_tube.rb
+- test/www/video_scraper/test_base.rb
 - test/www/test_video_scraper.rb
 - lib/www
 - lib/www/video_scraper
@@ -67,6 +70,7 @@ files:
 - lib/www/video_scraper/ameba_vision.rb
 - lib/www/video_scraper/age_sage.rb
 - lib/www/video_scraper/pornotube.rb
+- lib/www/video_scraper/adult_satellites.rb
 - lib/www/video_scraper/you_tube.rb
 - lib/www/video_scraper/moro_tube.rb
 - lib/www/video_scraper/pornhub.rb