valda-video_scraper 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/ChangeLog +4 -0
- data/README +69 -0
- data/Rakefile +144 -0
- data/lib/www/video_scraper/age_sage.rb +34 -0
- data/lib/www/video_scraper/ameba_vision.rb +28 -0
- data/lib/www/video_scraper/base.rb +60 -0
- data/lib/www/video_scraper/dailymotion.rb +36 -0
- data/lib/www/video_scraper/moro_tube.rb +37 -0
- data/lib/www/video_scraper/nico_video.rb +73 -0
- data/lib/www/video_scraper/pornhub.rb +29 -0
- data/lib/www/video_scraper/pornotube.rb +42 -0
- data/lib/www/video_scraper/red_tube.rb +62 -0
- data/lib/www/video_scraper/veoh.rb +30 -0
- data/lib/www/video_scraper/you_porn.rb +32 -0
- data/lib/www/video_scraper/you_tube.rb +55 -0
- data/lib/www/video_scraper/your_file_host.rb +60 -0
- data/lib/www/video_scraper.rb +85 -0
- data/test/test_helper.rb +8 -0
- data/test/www/test_video_scraper.rb +20 -0
- data/test/www/video_scraper/test_age_sage.rb +24 -0
- data/test/www/video_scraper/test_ameba_vision.rb +23 -0
- data/test/www/video_scraper/test_dailymotion.rb +25 -0
- data/test/www/video_scraper/test_moro_tube.rb +24 -0
- data/test/www/video_scraper/test_nico_video.rb +30 -0
- data/test/www/video_scraper/test_pornhub.rb +24 -0
- data/test/www/video_scraper/test_pornotube.rb +32 -0
- data/test/www/video_scraper/test_red_tube.rb +24 -0
- data/test/www/video_scraper/test_veoh.rb +24 -0
- data/test/www/video_scraper/test_you_porn.rb +24 -0
- data/test/www/video_scraper/test_you_tube.rb +40 -0
- data/test/www/video_scraper/test_your_file_host.rb +24 -0
- metadata +107 -0
@@ -0,0 +1,32 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.expand_path(File.dirname(__FILE__) + '/base')
|
4
|
+
|
5
|
+
module WWW
|
6
|
+
module VideoScraper
|
7
|
+
class YouPorn < Base
|
8
|
+
url_regex %r!\Ahttp://youporn\.com/watch/(\d+)!
|
9
|
+
|
10
|
+
def initialize(url, opt = nil)
|
11
|
+
super
|
12
|
+
do_query
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
def do_query
|
17
|
+
id = url_regex_match[1]
|
18
|
+
|
19
|
+
request_url = @page_url.sub(/(\?.*)?$/, '?user_choice=Enter')
|
20
|
+
html = http_get(request_url, 'Cookie' => 'age_check=1')
|
21
|
+
doc = Hpricot(html)
|
22
|
+
doc.search('//div[@id="download"]//a').each do |elem|
|
23
|
+
href = elem.attributes['href']
|
24
|
+
(@video_url = href; break) if href =~ %r!^http://download\.youporn\.com/download/.*\.flv!
|
25
|
+
end
|
26
|
+
h1 = doc.at('//div[@id="videoArea"]/h1')
|
27
|
+
@title = h1.inner_html.gsub(/<[^>]*>/, '').strip
|
28
|
+
@thumb_url = h1.at('/img').attributes['src'].sub(/(\d+)_small\.jpg$/, '\1_large.jpg')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.expand_path(File.dirname(__FILE__) + '/base')
|
4
|
+
|
5
|
+
module WWW
|
6
|
+
module VideoScraper
|
7
|
+
class YouTube < Base
|
8
|
+
url_regex %r!\Ahttp://(?:www|jp)\.youtube\.com/watch.*[?&]v=([[:alnum:]]+)!
|
9
|
+
|
10
|
+
def initialize(url, opt = nil)
|
11
|
+
super
|
12
|
+
do_query
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
def login
|
17
|
+
uri = URI.parse(@page_url)
|
18
|
+
page = agent.get("#{uri.scheme}://#{uri.host}/login")
|
19
|
+
login_form = page.forms.with.name('loginForm').first
|
20
|
+
login_form.username = @opt[:you_tube_username]
|
21
|
+
login_form.password = @opt[:you_tube_password]
|
22
|
+
agent.submit(login_form)
|
23
|
+
end
|
24
|
+
|
25
|
+
def pass_verify_age
|
26
|
+
uri = URI.parse(@page_url)
|
27
|
+
page = agent.get(uri)
|
28
|
+
if page.uri.path =~ /verify_age/
|
29
|
+
login
|
30
|
+
page = agent.post(page.uri,
|
31
|
+
'next_url' => "#{uri.path}?#{uri.query}",
|
32
|
+
'action_confirm' => 'Confirm Birth Date')
|
33
|
+
end
|
34
|
+
page
|
35
|
+
end
|
36
|
+
|
37
|
+
def do_query
|
38
|
+
page = pass_verify_age
|
39
|
+
@title = page.root.at('//head/title').inner_html.sub(/^YouTube[\s-]*/, '') rescue ''
|
40
|
+
@embed_tag = page.root.at('//input[@id="embed_code"]').attributes['value'] rescue nil
|
41
|
+
page.root.search('//script').each do |script|
|
42
|
+
if m = script.inner_html.match(/var\s+swfArgs\s*=\s*([^;]+);/)
|
43
|
+
swf_args = JSON::parse(m[1])
|
44
|
+
uri = URI.parse(@page_url)
|
45
|
+
uri.path = '/get_video'
|
46
|
+
uri.query = "video_id=#{swf_args['video_id']}&t=#{swf_args['t']}"
|
47
|
+
@video_url = uri.to_s
|
48
|
+
@thumb_url = "http://i.ytimg.com/vi/#{swf_args['video_id']}/default.jpg"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
raise FileNotFound, 'file not found' if @video_url.nil?
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.expand_path(File.dirname(__FILE__) + '/base')
|
4
|
+
|
5
|
+
module WWW
|
6
|
+
module VideoScraper
|
7
|
+
class YourFileHost < Base
|
8
|
+
url_regex %r!\Ahttp://www\.yourfilehost\.com/media\.php\?cat=video&file=.+$!
|
9
|
+
|
10
|
+
class MaximumVideoPlaysReached < TryAgainLater; end
|
11
|
+
class BandwidthAllowanceExceeded < TryAgainLater; end
|
12
|
+
class NoFileCategory < FileNotFound; end
|
13
|
+
|
14
|
+
def initialize(url, opt = nil)
|
15
|
+
super
|
16
|
+
do_query
|
17
|
+
end
|
18
|
+
|
19
|
+
def filename
|
20
|
+
uri = URI.parse(@page_url)
|
21
|
+
q = CGI.parse(uri.query)
|
22
|
+
q['file'][0]
|
23
|
+
end
|
24
|
+
alias :title :filename
|
25
|
+
|
26
|
+
private
|
27
|
+
def do_query
|
28
|
+
html = http_get(@page_url)
|
29
|
+
doc = Hpricot(html.toutf8)
|
30
|
+
if elem = doc.at('//object[@id="objectPlayer"] //param[@name="movie"]')
|
31
|
+
value = elem.attributes['value']
|
32
|
+
raise StandardError, 'video information is not found' unless value
|
33
|
+
v = CGI::parse(value)
|
34
|
+
if request_url = v['video'][0]
|
35
|
+
response_body = http_get(request_url)
|
36
|
+
q = CGI::parse(response_body)
|
37
|
+
@thumb_url = q['photo'][0] rescue ''
|
38
|
+
@video_url = q['video_id'][0] rescue ''
|
39
|
+
end
|
40
|
+
elsif elem = doc.at('//object[@id="VIDEO"] //param[@name="URL"]')
|
41
|
+
@video_url = elem.attributes['value']
|
42
|
+
else
|
43
|
+
if html =~ /MAXIMUM VIDEO PLAYS REACHED/i
|
44
|
+
raise MaximumVideoPlaysReached, 'MAXIMUM VIDEO PLAYS REACHED'
|
45
|
+
elsif html =~ /Bandwidth Allowance exceeded/i
|
46
|
+
raise BandwidthAllowanceExceeded, 'Bandwidth Allowance exceeded'
|
47
|
+
elsif html =~ /url=error\.php\?err=8/i
|
48
|
+
raise FileNotFound, 'file not found'
|
49
|
+
elsif html =~ /url=error\.php\?err=5/i or html =~ /no file category/i
|
50
|
+
raise NoFileCategory, 'no file category'
|
51
|
+
elsif html =~ /File not found/i
|
52
|
+
raise FileNotFound, 'file not found'
|
53
|
+
else
|
54
|
+
raise TryAgainLater, 'scrape failed'
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'open-uri'
|
5
|
+
require 'mechanize'
|
6
|
+
require 'kconv'
|
7
|
+
require 'json'
|
8
|
+
require 'uri'
|
9
|
+
begin
|
10
|
+
require 'cgialt'
|
11
|
+
rescue LoadError
|
12
|
+
require 'cgi'
|
13
|
+
end
|
14
|
+
|
15
|
+
module WWW
|
16
|
+
module VideoScraper
|
17
|
+
class TryAgainLater < RuntimeError; end
|
18
|
+
class FileNotFound < RuntimeError; end
|
19
|
+
VERSION = '1.0.1'
|
20
|
+
|
21
|
+
MODULES_NAME = %w(age_sage ameba_vision dailymotion moro_tube
|
22
|
+
nico_video pornhub pornotube red_tube veoh
|
23
|
+
you_porn you_tube your_file_host)
|
24
|
+
|
25
|
+
@@modules = MODULES_NAME.map do |name|
|
26
|
+
require File.expand_path(File.join(File.dirname(__FILE__), 'video_scraper', name))
|
27
|
+
const_get( name.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase } )
|
28
|
+
end
|
29
|
+
|
30
|
+
@@options = {
|
31
|
+
:logger => nil,
|
32
|
+
:cache => nil,
|
33
|
+
:debug => false,
|
34
|
+
}
|
35
|
+
|
36
|
+
class << self
|
37
|
+
def options
|
38
|
+
@@options
|
39
|
+
end
|
40
|
+
|
41
|
+
def options=(opts)
|
42
|
+
@@options = opts
|
43
|
+
end
|
44
|
+
|
45
|
+
def configure(&proc)
|
46
|
+
raise ArgumentError, "Block is required." unless block_given?
|
47
|
+
yield @@options
|
48
|
+
end
|
49
|
+
|
50
|
+
# 与えられた URL を処理できるモジュールを @@modules から検索して実行する
|
51
|
+
def scrape(url, opt = nil)
|
52
|
+
opt = @@options.merge(opt || {})
|
53
|
+
opt[:logger] ||= logger
|
54
|
+
raise StandardError, "url param is requred" unless url
|
55
|
+
|
56
|
+
@@modules.each do |scraper|
|
57
|
+
if scraper.valid_url?(url)
|
58
|
+
logger.info "scraper: #{scraper.to_s}"
|
59
|
+
logger.info "url: #{url}"
|
60
|
+
return scraper.new(url, opt)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
logger.info "unsupport site."
|
64
|
+
return nil
|
65
|
+
rescue TimeoutError, Timeout::Error, Errno::ETIMEDOUT => e
|
66
|
+
logger.warn " Timeout : #{e.to_s}"
|
67
|
+
raise TryAgainLater, e.to_s
|
68
|
+
rescue OpenURI::HTTPError => e
|
69
|
+
raise TryAgainLater, e.to_s if e.to_s.match(/50\d/)
|
70
|
+
raise FileNotFound, e.to_s if e.to_s.match(/40\d/)
|
71
|
+
raise
|
72
|
+
rescue Exception => e
|
73
|
+
logger.error "#{e.class}: #{e.to_s}"
|
74
|
+
raise e
|
75
|
+
end
|
76
|
+
|
77
|
+
private
|
78
|
+
def logger
|
79
|
+
return @@options[:logger] if @@options[:logger]
|
80
|
+
require 'logger'
|
81
|
+
@@options[:logger] = Logger.new(STDOUT)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../test_helper'
|
4
|
+
|
5
|
+
class TestVideoScraper < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
end
|
9
|
+
|
10
|
+
def teardown
|
11
|
+
FileUtils.remove_entry_secure(@cache_root, true)
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_configure
|
15
|
+
WWW::VideoScraper.configure do |conf|
|
16
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
17
|
+
end
|
18
|
+
assert_kind_of FileCache, WWW::VideoScraper.options[:cache]
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestAgeSage < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://adult.agesage.jp/contentsPage.html?mcd=oadyChZgcoN9rqbJ')
|
19
|
+
assert_equal 'http://adult.agesage.jp/contentsPage.html?mcd=oadyChZgcoN9rqbJ', vs.page_url
|
20
|
+
assert_match %r!http://file\d+\.agesage\.jp/flv/\d{8}/\d{10}_\d{6}\.flv!, vs.video_url
|
21
|
+
assert_match %r!http://file\d+\.agesage\.jp/data/\d{8}/\d{10}_\d{6}!, vs.thumb_url
|
22
|
+
assert_match %r!^<script type=\"text/javascript\" src=\".*</script>$!, vs.embed_tag
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestAmebaVision < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://vision.ameba.jp/watch.do?movie=772341')
|
19
|
+
assert_equal 'http://vision.ameba.jp/watch.do?movie=772341', vs.page_url
|
20
|
+
assert_match %r|http://[-[:alnum:]]+\.vision\.ameba\.jp/flv/\d{4}/\d{2}/\d{2}/[[:alnum:]]+\.flv|, vs.video_url
|
21
|
+
assert_match %r|http://[-[:alnum:]]+\.vision\.ameba\.jp/jpg/\d{4}/\d{2}/\d{2}/[[:alnum:]]+_4\.jpg|, vs.thumb_url
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestDailymotion < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://www.dailymotion.com/mokelov/japan/video/x12gr3_music')
|
19
|
+
assert_equal 'http://www.dailymotion.com/mokelov/japan/video/x12gr3_music', vs.page_url
|
20
|
+
assert_match %r|http://www\.dailymotion\.com/get/\d{2}/\d+x\d+/flv/\d+\.flv\?key=[[:alnum:]]+|, vs.video_url
|
21
|
+
assert_match %r|http://\w+\.static\.dailymotion\.com/dyn/preview/\d+x\d+/\d+\.jpg\?\d{14}|, vs.thumb_url
|
22
|
+
assert_match %r|^<div><object\s.*</div>$|, vs.embed_tag
|
23
|
+
assert_equal '遭難 東京事変', vs.title
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestMoroTube < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://www.morotube.com/watch.php?clip=46430e1d')
|
19
|
+
assert_equal 'http://www.morotube.com/watch.php?clip=46430e1d', vs.page_url
|
20
|
+
assert_match %r!http://video\d+\.morotube\.com/[[:alnum:]]{32}/[[:alnum:]]{8}/[[:alnum:]]{8}\.flv!, vs.video_url
|
21
|
+
assert_match %r!http://static\d+\.morotube\.com/thumbs/\w{8}\.jpg!, vs.thumb_url
|
22
|
+
assert_match %r!^<embed\s.*</embed>$!, vs.embed_tag
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestNicoVideo < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
config = Pit.get('nicovideo.jp', :require => {
|
12
|
+
'mail' => 'your email in nicovideo.jp',
|
13
|
+
'password' => 'your password in nicovideo.jp'
|
14
|
+
})
|
15
|
+
@opt = { :nico_video_mail => config['mail'], :nico_video_password => config['password'] }
|
16
|
+
end
|
17
|
+
|
18
|
+
def teardown
|
19
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_scrape
|
23
|
+
vs = WWW::VideoScraper.scrape('http://www.nicovideo.jp/watch/sm1175788', @opt)
|
24
|
+
assert_equal 'http://www.nicovideo.jp/watch/sm1175788', vs.page_url
|
25
|
+
assert_match %r|http://smile-[[:alnum:]]+\.nicovideo\.jp/smile\?v=\d{7}\.\d{5}|, vs.video_url
|
26
|
+
assert_match %r|http://tn-skr\d+\.smilevideo\.jp/smile\?i=\d{7}|, vs.thumb_url
|
27
|
+
assert_match %r|^<iframe\s+.*</iframe>$|, vs.embed_tag
|
28
|
+
assert_equal '本格的 ガチムチパンツレスリング', vs.title
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestPornhub < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://www.pornhub.com/view_video.php?viewkey=27f115e7fee8c18f92b0')
|
19
|
+
assert_equal 'http://www.pornhub.com/view_video.php?viewkey=27f115e7fee8c18f92b0', vs.page_url
|
20
|
+
assert_match %r|http://media1.pornhub.com/dl/[[:alnum:]]{32}/[[:alnum:]]{8}/videos/000/191/743/191743\.flv|, vs.video_url
|
21
|
+
assert_equal 'http://p1.pornhub.com/thumbs/000/191/743/small.jpg', vs.thumb_url
|
22
|
+
assert_match %r|^<object type=\"application/x-shockwave-flash\" data=\".*</object>$|, vs.embed_tag
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestPornotube < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://pornotube.com/media.php?m=1677937')
|
19
|
+
assert_equal 'http://pornotube.com/media.php?m=1677937', vs.page_url
|
20
|
+
assert_match %r|http://video\d+\.pornotube\.com/\d+/\d+\.flv|, vs.video_url
|
21
|
+
assert_match %r|http://photo\.pornotube\.com/thumbnails/video/\d+/\d+\.jpg|, vs.thumb_url
|
22
|
+
assert_match %r|^<embed\s+src=\".*\"\s*/>$|, vs.embed_tag
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_scrape_alt_url
|
26
|
+
vs = WWW::VideoScraper.scrape('http://pornotube.com/channels.php?channelId=83&m=1677912')
|
27
|
+
assert_equal 'http://pornotube.com/channels.php?channelId=83&m=1677912', vs.page_url
|
28
|
+
assert_match %r|http://video\d+\.pornotube\.com/\d+/\d+\.flv|, vs.video_url
|
29
|
+
assert_match %r|http://photo\.pornotube\.com/thumbnails/video/\d+/\d+\.jpg|, vs.thumb_url
|
30
|
+
assert_match %r|^<embed\s+src=\".*\"\s*/>$|, vs.embed_tag
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestRedTube < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://www.redtube.com/8415')
|
19
|
+
assert_equal 'http://www.redtube.com/8415', vs.page_url
|
20
|
+
assert_match %r!http://dl\.redtube\.com/_videos_t4vn23s9jc5498tgj49icfj4678/0000008/Z2XDJA1ZL\.flv!, vs.video_url
|
21
|
+
assert_nil vs.thumb_url
|
22
|
+
assert_match %r!<object\s+.*</object>!, vs.embed_tag
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestVeoh < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://www.veoh.com/videos/v6245232rh8aGEM9')
|
19
|
+
assert_equal 'http://www.veoh.com/videos/v6245232rh8aGEM9', vs.page_url
|
20
|
+
assert_match %r|http://content\.veoh\.com/flash/p/\d/[[:alnum:]]{16}/[[:alnum:]]{40}\.fll\?ct=[[:alnum:]]{48}|, vs.video_url
|
21
|
+
assert_match %r|http://p-images\.veoh\.com/image\.out\?imageId=media-[[:alnum:]]+.jpg|, vs.thumb_url
|
22
|
+
assert_match %r|^<embed\s.*>$|, vs.embed_tag
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestYouPorn < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://youporn.com/watch/93495?user_choice=Enter')
|
19
|
+
assert_equal 'http://youporn.com/watch/93495?user_choice=Enter', vs.page_url
|
20
|
+
assert_match %r|http://download\.youporn\.com/download/\d+/flv/\d+_.*\.flv.*|, vs.video_url
|
21
|
+
assert_match %r|http://ss-\d+\.youporn\.com/screenshot/\d+/\d+/screenshot/\d+_large\.jpg|, vs.thumb_url
|
22
|
+
assert_equal 'unbelievable Handjob!', vs.title
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestYouTube < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
|
12
|
+
config = Pit.get('youtube.com', :require => {
|
13
|
+
'username' => 'your email in youtube.com',
|
14
|
+
'password' => 'your password in youtube.com'
|
15
|
+
})
|
16
|
+
@opt = { :you_tube_username => config['username'], :you_tube_password => config['password'] }
|
17
|
+
end
|
18
|
+
|
19
|
+
def teardown
|
20
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_scrape
|
24
|
+
vs = WWW::VideoScraper.scrape('http://jp.youtube.com/watch?v=Ym20IwIUbuU', @opt)
|
25
|
+
assert_equal 'http://jp.youtube.com/watch?v=Ym20IwIUbuU', vs.page_url
|
26
|
+
assert_match %r|http://jp\.youtube\.com/get_video\?video_id=Ym20IwIUbuU&t=[-_[:alnum:]]{32}|, vs.video_url
|
27
|
+
assert_match %r|http://\w\.ytimg\.com/vi/Ym20IwIUbuU/default\.jpg|, vs.thumb_url
|
28
|
+
assert_match %r|^<object\s+.*</object>$|, vs.embed_tag
|
29
|
+
assert_equal 'とらドラ!Toradora ep15 2-3', vs.title
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_scrape_alt_url
|
33
|
+
vs = WWW::VideoScraper.scrape('http://jp.youtube.com/watch?v=ibhaQZB9TWU', @opt)
|
34
|
+
assert_equal 'http://jp.youtube.com/watch?v=ibhaQZB9TWU', vs.page_url
|
35
|
+
assert_match %r|http://jp\.youtube\.com/get_video\?video_id=ibhaQZB9TWU&t=[-_[:alnum:]]{32}|, vs.video_url
|
36
|
+
assert_match %r|http://\w\.ytimg\.com/vi/ibhaQZB9TWU/default\.jpg|, vs.thumb_url
|
37
|
+
assert_match %r|^<object\s+.*</object>$|, vs.embed_tag
|
38
|
+
assert_equal '水着の人妻', vs.title
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestYourFileHost < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv')
|
19
|
+
assert_equal 'http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv', vs.page_url
|
20
|
+
assert_match %r|http://cdn\w*\.yourfilehost\.com/unit1/flash8/\d+/[[:alnum:]]{32}\.flv|, vs.video_url
|
21
|
+
assert_match %r|http://cdn\w*\.yourfilehost\.com/thumbs/\d+/[[:alnum:]]{32}\.jpg|, vs.thumb_url
|
22
|
+
assert_equal 'XV436__03.wmv', vs.title
|
23
|
+
end
|
24
|
+
end
|