valda-video_scraper 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/ChangeLog +4 -0
- data/README +69 -0
- data/Rakefile +144 -0
- data/lib/www/video_scraper/age_sage.rb +34 -0
- data/lib/www/video_scraper/ameba_vision.rb +28 -0
- data/lib/www/video_scraper/base.rb +60 -0
- data/lib/www/video_scraper/dailymotion.rb +36 -0
- data/lib/www/video_scraper/moro_tube.rb +37 -0
- data/lib/www/video_scraper/nico_video.rb +73 -0
- data/lib/www/video_scraper/pornhub.rb +29 -0
- data/lib/www/video_scraper/pornotube.rb +42 -0
- data/lib/www/video_scraper/red_tube.rb +62 -0
- data/lib/www/video_scraper/veoh.rb +30 -0
- data/lib/www/video_scraper/you_porn.rb +32 -0
- data/lib/www/video_scraper/you_tube.rb +55 -0
- data/lib/www/video_scraper/your_file_host.rb +60 -0
- data/lib/www/video_scraper.rb +85 -0
- data/test/test_helper.rb +8 -0
- data/test/www/test_video_scraper.rb +20 -0
- data/test/www/video_scraper/test_age_sage.rb +24 -0
- data/test/www/video_scraper/test_ameba_vision.rb +23 -0
- data/test/www/video_scraper/test_dailymotion.rb +25 -0
- data/test/www/video_scraper/test_moro_tube.rb +24 -0
- data/test/www/video_scraper/test_nico_video.rb +30 -0
- data/test/www/video_scraper/test_pornhub.rb +24 -0
- data/test/www/video_scraper/test_pornotube.rb +32 -0
- data/test/www/video_scraper/test_red_tube.rb +24 -0
- data/test/www/video_scraper/test_veoh.rb +24 -0
- data/test/www/video_scraper/test_you_porn.rb +24 -0
- data/test/www/video_scraper/test_you_tube.rb +40 -0
- data/test/www/video_scraper/test_your_file_host.rb +24 -0
- metadata +107 -0
@@ -0,0 +1,32 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.expand_path(File.dirname(__FILE__) + '/base')
|
4
|
+
|
5
|
+
module WWW
|
6
|
+
module VideoScraper
|
7
|
+
class YouPorn < Base
|
8
|
+
url_regex %r!\Ahttp://youporn\.com/watch/(\d+)!
|
9
|
+
|
10
|
+
def initialize(url, opt = nil)
|
11
|
+
super
|
12
|
+
do_query
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
def do_query
|
17
|
+
id = url_regex_match[1]
|
18
|
+
|
19
|
+
request_url = @page_url.sub(/(\?.*)?$/, '?user_choice=Enter')
|
20
|
+
html = http_get(request_url, 'Cookie' => 'age_check=1')
|
21
|
+
doc = Hpricot(html)
|
22
|
+
doc.search('//div[@id="download"]//a').each do |elem|
|
23
|
+
href = elem.attributes['href']
|
24
|
+
(@video_url = href; break) if href =~ %r!^http://download\.youporn\.com/download/.*\.flv!
|
25
|
+
end
|
26
|
+
h1 = doc.at('//div[@id="videoArea"]/h1')
|
27
|
+
@title = h1.inner_html.gsub(/<[^>]*>/, '').strip
|
28
|
+
@thumb_url = h1.at('/img').attributes['src'].sub(/(\d+)_small\.jpg$/, '\1_large.jpg')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.expand_path(File.dirname(__FILE__) + '/base')
|
4
|
+
|
5
|
+
module WWW
|
6
|
+
module VideoScraper
|
7
|
+
class YouTube < Base
|
8
|
+
url_regex %r!\Ahttp://(?:www|jp)\.youtube\.com/watch.*[?&]v=([[:alnum:]]+)!
|
9
|
+
|
10
|
+
def initialize(url, opt = nil)
|
11
|
+
super
|
12
|
+
do_query
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
def login
|
17
|
+
uri = URI.parse(@page_url)
|
18
|
+
page = agent.get("#{uri.scheme}://#{uri.host}/login")
|
19
|
+
login_form = page.forms.with.name('loginForm').first
|
20
|
+
login_form.username = @opt[:you_tube_username]
|
21
|
+
login_form.password = @opt[:you_tube_password]
|
22
|
+
agent.submit(login_form)
|
23
|
+
end
|
24
|
+
|
25
|
+
def pass_verify_age
|
26
|
+
uri = URI.parse(@page_url)
|
27
|
+
page = agent.get(uri)
|
28
|
+
if page.uri.path =~ /verify_age/
|
29
|
+
login
|
30
|
+
page = agent.post(page.uri,
|
31
|
+
'next_url' => "#{uri.path}?#{uri.query}",
|
32
|
+
'action_confirm' => 'Confirm Birth Date')
|
33
|
+
end
|
34
|
+
page
|
35
|
+
end
|
36
|
+
|
37
|
+
def do_query
|
38
|
+
page = pass_verify_age
|
39
|
+
@title = page.root.at('//head/title').inner_html.sub(/^YouTube[\s-]*/, '') rescue ''
|
40
|
+
@embed_tag = page.root.at('//input[@id="embed_code"]').attributes['value'] rescue nil
|
41
|
+
page.root.search('//script').each do |script|
|
42
|
+
if m = script.inner_html.match(/var\s+swfArgs\s*=\s*([^;]+);/)
|
43
|
+
swf_args = JSON::parse(m[1])
|
44
|
+
uri = URI.parse(@page_url)
|
45
|
+
uri.path = '/get_video'
|
46
|
+
uri.query = "video_id=#{swf_args['video_id']}&t=#{swf_args['t']}"
|
47
|
+
@video_url = uri.to_s
|
48
|
+
@thumb_url = "http://i.ytimg.com/vi/#{swf_args['video_id']}/default.jpg"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
raise FileNotFound, 'file not found' if @video_url.nil?
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.expand_path(File.dirname(__FILE__) + '/base')
|
4
|
+
|
5
|
+
module WWW
|
6
|
+
module VideoScraper
|
7
|
+
class YourFileHost < Base
|
8
|
+
url_regex %r!\Ahttp://www\.yourfilehost\.com/media\.php\?cat=video&file=.+$!
|
9
|
+
|
10
|
+
class MaximumVideoPlaysReached < TryAgainLater; end
|
11
|
+
class BandwidthAllowanceExceeded < TryAgainLater; end
|
12
|
+
class NoFileCategory < FileNotFound; end
|
13
|
+
|
14
|
+
def initialize(url, opt = nil)
|
15
|
+
super
|
16
|
+
do_query
|
17
|
+
end
|
18
|
+
|
19
|
+
def filename
|
20
|
+
uri = URI.parse(@page_url)
|
21
|
+
q = CGI.parse(uri.query)
|
22
|
+
q['file'][0]
|
23
|
+
end
|
24
|
+
alias :title :filename
|
25
|
+
|
26
|
+
private
|
27
|
+
def do_query
|
28
|
+
html = http_get(@page_url)
|
29
|
+
doc = Hpricot(html.toutf8)
|
30
|
+
if elem = doc.at('//object[@id="objectPlayer"] //param[@name="movie"]')
|
31
|
+
value = elem.attributes['value']
|
32
|
+
raise StandardError, 'video information is not found' unless value
|
33
|
+
v = CGI::parse(value)
|
34
|
+
if request_url = v['video'][0]
|
35
|
+
response_body = http_get(request_url)
|
36
|
+
q = CGI::parse(response_body)
|
37
|
+
@thumb_url = q['photo'][0] rescue ''
|
38
|
+
@video_url = q['video_id'][0] rescue ''
|
39
|
+
end
|
40
|
+
elsif elem = doc.at('//object[@id="VIDEO"] //param[@name="URL"]')
|
41
|
+
@video_url = elem.attributes['value']
|
42
|
+
else
|
43
|
+
if html =~ /MAXIMUM VIDEO PLAYS REACHED/i
|
44
|
+
raise MaximumVideoPlaysReached, 'MAXIMUM VIDEO PLAYS REACHED'
|
45
|
+
elsif html =~ /Bandwidth Allowance exceeded/i
|
46
|
+
raise BandwidthAllowanceExceeded, 'Bandwidth Allowance exceeded'
|
47
|
+
elsif html =~ /url=error\.php\?err=8/i
|
48
|
+
raise FileNotFound, 'file not found'
|
49
|
+
elsif html =~ /url=error\.php\?err=5/i or html =~ /no file category/i
|
50
|
+
raise NoFileCategory, 'no file category'
|
51
|
+
elsif html =~ /File not found/i
|
52
|
+
raise FileNotFound, 'file not found'
|
53
|
+
else
|
54
|
+
raise TryAgainLater, 'scrape failed'
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'open-uri'
|
5
|
+
require 'mechanize'
|
6
|
+
require 'kconv'
|
7
|
+
require 'json'
|
8
|
+
require 'uri'
|
9
|
+
begin
|
10
|
+
require 'cgialt'
|
11
|
+
rescue LoadError
|
12
|
+
require 'cgi'
|
13
|
+
end
|
14
|
+
|
15
|
+
module WWW
|
16
|
+
module VideoScraper
|
17
|
+
class TryAgainLater < RuntimeError; end
|
18
|
+
class FileNotFound < RuntimeError; end
|
19
|
+
VERSION = '1.0.1'
|
20
|
+
|
21
|
+
MODULES_NAME = %w(age_sage ameba_vision dailymotion moro_tube
|
22
|
+
nico_video pornhub pornotube red_tube veoh
|
23
|
+
you_porn you_tube your_file_host)
|
24
|
+
|
25
|
+
@@modules = MODULES_NAME.map do |name|
|
26
|
+
require File.expand_path(File.join(File.dirname(__FILE__), 'video_scraper', name))
|
27
|
+
const_get( name.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase } )
|
28
|
+
end
|
29
|
+
|
30
|
+
@@options = {
|
31
|
+
:logger => nil,
|
32
|
+
:cache => nil,
|
33
|
+
:debug => false,
|
34
|
+
}
|
35
|
+
|
36
|
+
class << self
|
37
|
+
def options
|
38
|
+
@@options
|
39
|
+
end
|
40
|
+
|
41
|
+
def options=(opts)
|
42
|
+
@@options = opts
|
43
|
+
end
|
44
|
+
|
45
|
+
def configure(&proc)
|
46
|
+
raise ArgumentError, "Block is required." unless block_given?
|
47
|
+
yield @@options
|
48
|
+
end
|
49
|
+
|
50
|
+
# 与えられた URL を処理できるモジュールを @@modules から検索して実行する
|
51
|
+
def scrape(url, opt = nil)
|
52
|
+
opt = @@options.merge(opt || {})
|
53
|
+
opt[:logger] ||= logger
|
54
|
+
raise StandardError, "url param is requred" unless url
|
55
|
+
|
56
|
+
@@modules.each do |scraper|
|
57
|
+
if scraper.valid_url?(url)
|
58
|
+
logger.info "scraper: #{scraper.to_s}"
|
59
|
+
logger.info "url: #{url}"
|
60
|
+
return scraper.new(url, opt)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
logger.info "unsupport site."
|
64
|
+
return nil
|
65
|
+
rescue TimeoutError, Timeout::Error, Errno::ETIMEDOUT => e
|
66
|
+
logger.warn " Timeout : #{e.to_s}"
|
67
|
+
raise TryAgainLater, e.to_s
|
68
|
+
rescue OpenURI::HTTPError => e
|
69
|
+
raise TryAgainLater, e.to_s if e.to_s.match(/50\d/)
|
70
|
+
raise FileNotFound, e.to_s if e.to_s.match(/40\d/)
|
71
|
+
raise
|
72
|
+
rescue Exception => e
|
73
|
+
logger.error "#{e.class}: #{e.to_s}"
|
74
|
+
raise e
|
75
|
+
end
|
76
|
+
|
77
|
+
private
|
78
|
+
def logger
|
79
|
+
return @@options[:logger] if @@options[:logger]
|
80
|
+
require 'logger'
|
81
|
+
@@options[:logger] = Logger.new(STDOUT)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../test_helper'
|
4
|
+
|
5
|
+
class TestVideoScraper < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
end
|
9
|
+
|
10
|
+
def teardown
|
11
|
+
FileUtils.remove_entry_secure(@cache_root, true)
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_configure
|
15
|
+
WWW::VideoScraper.configure do |conf|
|
16
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
17
|
+
end
|
18
|
+
assert_kind_of FileCache, WWW::VideoScraper.options[:cache]
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestAgeSage < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://adult.agesage.jp/contentsPage.html?mcd=oadyChZgcoN9rqbJ')
|
19
|
+
assert_equal 'http://adult.agesage.jp/contentsPage.html?mcd=oadyChZgcoN9rqbJ', vs.page_url
|
20
|
+
assert_match %r!http://file\d+\.agesage\.jp/flv/\d{8}/\d{10}_\d{6}\.flv!, vs.video_url
|
21
|
+
assert_match %r!http://file\d+\.agesage\.jp/data/\d{8}/\d{10}_\d{6}!, vs.thumb_url
|
22
|
+
assert_match %r!^<script type=\"text/javascript\" src=\".*</script>$!, vs.embed_tag
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestAmebaVision < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://vision.ameba.jp/watch.do?movie=772341')
|
19
|
+
assert_equal 'http://vision.ameba.jp/watch.do?movie=772341', vs.page_url
|
20
|
+
assert_match %r|http://[-[:alnum:]]+\.vision\.ameba\.jp/flv/\d{4}/\d{2}/\d{2}/[[:alnum:]]+\.flv|, vs.video_url
|
21
|
+
assert_match %r|http://[-[:alnum:]]+\.vision\.ameba\.jp/jpg/\d{4}/\d{2}/\d{2}/[[:alnum:]]+_4\.jpg|, vs.thumb_url
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestDailymotion < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://www.dailymotion.com/mokelov/japan/video/x12gr3_music')
|
19
|
+
assert_equal 'http://www.dailymotion.com/mokelov/japan/video/x12gr3_music', vs.page_url
|
20
|
+
assert_match %r|http://www\.dailymotion\.com/get/\d{2}/\d+x\d+/flv/\d+\.flv\?key=[[:alnum:]]+|, vs.video_url
|
21
|
+
assert_match %r|http://\w+\.static\.dailymotion\.com/dyn/preview/\d+x\d+/\d+\.jpg\?\d{14}|, vs.thumb_url
|
22
|
+
assert_match %r|^<div><object\s.*</div>$|, vs.embed_tag
|
23
|
+
assert_equal '遭難 東京事変', vs.title
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestMoroTube < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://www.morotube.com/watch.php?clip=46430e1d')
|
19
|
+
assert_equal 'http://www.morotube.com/watch.php?clip=46430e1d', vs.page_url
|
20
|
+
assert_match %r!http://video\d+\.morotube\.com/[[:alnum:]]{32}/[[:alnum:]]{8}/[[:alnum:]]{8}\.flv!, vs.video_url
|
21
|
+
assert_match %r!http://static\d+\.morotube\.com/thumbs/\w{8}\.jpg!, vs.thumb_url
|
22
|
+
assert_match %r!^<embed\s.*</embed>$!, vs.embed_tag
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestNicoVideo < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
config = Pit.get('nicovideo.jp', :require => {
|
12
|
+
'mail' => 'your email in nicovideo.jp',
|
13
|
+
'password' => 'your password in nicovideo.jp'
|
14
|
+
})
|
15
|
+
@opt = { :nico_video_mail => config['mail'], :nico_video_password => config['password'] }
|
16
|
+
end
|
17
|
+
|
18
|
+
def teardown
|
19
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_scrape
|
23
|
+
vs = WWW::VideoScraper.scrape('http://www.nicovideo.jp/watch/sm1175788', @opt)
|
24
|
+
assert_equal 'http://www.nicovideo.jp/watch/sm1175788', vs.page_url
|
25
|
+
assert_match %r|http://smile-[[:alnum:]]+\.nicovideo\.jp/smile\?v=\d{7}\.\d{5}|, vs.video_url
|
26
|
+
assert_match %r|http://tn-skr\d+\.smilevideo\.jp/smile\?i=\d{7}|, vs.thumb_url
|
27
|
+
assert_match %r|^<iframe\s+.*</iframe>$|, vs.embed_tag
|
28
|
+
assert_equal '本格的 ガチムチパンツレスリング', vs.title
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestPornhub < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://www.pornhub.com/view_video.php?viewkey=27f115e7fee8c18f92b0')
|
19
|
+
assert_equal 'http://www.pornhub.com/view_video.php?viewkey=27f115e7fee8c18f92b0', vs.page_url
|
20
|
+
assert_match %r|http://media1.pornhub.com/dl/[[:alnum:]]{32}/[[:alnum:]]{8}/videos/000/191/743/191743\.flv|, vs.video_url
|
21
|
+
assert_equal 'http://p1.pornhub.com/thumbs/000/191/743/small.jpg', vs.thumb_url
|
22
|
+
assert_match %r|^<object type=\"application/x-shockwave-flash\" data=\".*</object>$|, vs.embed_tag
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestPornotube < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://pornotube.com/media.php?m=1677937')
|
19
|
+
assert_equal 'http://pornotube.com/media.php?m=1677937', vs.page_url
|
20
|
+
assert_match %r|http://video\d+\.pornotube\.com/\d+/\d+\.flv|, vs.video_url
|
21
|
+
assert_match %r|http://photo\.pornotube\.com/thumbnails/video/\d+/\d+\.jpg|, vs.thumb_url
|
22
|
+
assert_match %r|^<embed\s+src=\".*\"\s*/>$|, vs.embed_tag
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_scrape_alt_url
|
26
|
+
vs = WWW::VideoScraper.scrape('http://pornotube.com/channels.php?channelId=83&m=1677912')
|
27
|
+
assert_equal 'http://pornotube.com/channels.php?channelId=83&m=1677912', vs.page_url
|
28
|
+
assert_match %r|http://video\d+\.pornotube\.com/\d+/\d+\.flv|, vs.video_url
|
29
|
+
assert_match %r|http://photo\.pornotube\.com/thumbnails/video/\d+/\d+\.jpg|, vs.thumb_url
|
30
|
+
assert_match %r|^<embed\s+src=\".*\"\s*/>$|, vs.embed_tag
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestRedTube < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://www.redtube.com/8415')
|
19
|
+
assert_equal 'http://www.redtube.com/8415', vs.page_url
|
20
|
+
assert_match %r!http://dl\.redtube\.com/_videos_t4vn23s9jc5498tgj49icfj4678/0000008/Z2XDJA1ZL\.flv!, vs.video_url
|
21
|
+
assert_nil vs.thumb_url
|
22
|
+
assert_match %r!<object\s+.*</object>!, vs.embed_tag
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestVeoh < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://www.veoh.com/videos/v6245232rh8aGEM9')
|
19
|
+
assert_equal 'http://www.veoh.com/videos/v6245232rh8aGEM9', vs.page_url
|
20
|
+
assert_match %r|http://content\.veoh\.com/flash/p/\d/[[:alnum:]]{16}/[[:alnum:]]{40}\.fll\?ct=[[:alnum:]]{48}|, vs.video_url
|
21
|
+
assert_match %r|http://p-images\.veoh\.com/image\.out\?imageId=media-[[:alnum:]]+.jpg|, vs.thumb_url
|
22
|
+
assert_match %r|^<embed\s.*>$|, vs.embed_tag
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestYouPorn < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://youporn.com/watch/93495?user_choice=Enter')
|
19
|
+
assert_equal 'http://youporn.com/watch/93495?user_choice=Enter', vs.page_url
|
20
|
+
assert_match %r|http://download\.youporn\.com/download/\d+/flv/\d+_.*\.flv.*|, vs.video_url
|
21
|
+
assert_match %r|http://ss-\d+\.youporn\.com/screenshot/\d+/\d+/screenshot/\d+_large\.jpg|, vs.thumb_url
|
22
|
+
assert_equal 'unbelievable Handjob!', vs.title
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestYouTube < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
|
12
|
+
config = Pit.get('youtube.com', :require => {
|
13
|
+
'username' => 'your email in youtube.com',
|
14
|
+
'password' => 'your password in youtube.com'
|
15
|
+
})
|
16
|
+
@opt = { :you_tube_username => config['username'], :you_tube_password => config['password'] }
|
17
|
+
end
|
18
|
+
|
19
|
+
def teardown
|
20
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_scrape
|
24
|
+
vs = WWW::VideoScraper.scrape('http://jp.youtube.com/watch?v=Ym20IwIUbuU', @opt)
|
25
|
+
assert_equal 'http://jp.youtube.com/watch?v=Ym20IwIUbuU', vs.page_url
|
26
|
+
assert_match %r|http://jp\.youtube\.com/get_video\?video_id=Ym20IwIUbuU&t=[-_[:alnum:]]{32}|, vs.video_url
|
27
|
+
assert_match %r|http://\w\.ytimg\.com/vi/Ym20IwIUbuU/default\.jpg|, vs.thumb_url
|
28
|
+
assert_match %r|^<object\s+.*</object>$|, vs.embed_tag
|
29
|
+
assert_equal 'とらドラ!Toradora ep15 2-3', vs.title
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_scrape_alt_url
|
33
|
+
vs = WWW::VideoScraper.scrape('http://jp.youtube.com/watch?v=ibhaQZB9TWU', @opt)
|
34
|
+
assert_equal 'http://jp.youtube.com/watch?v=ibhaQZB9TWU', vs.page_url
|
35
|
+
assert_match %r|http://jp\.youtube\.com/get_video\?video_id=ibhaQZB9TWU&t=[-_[:alnum:]]{32}|, vs.video_url
|
36
|
+
assert_match %r|http://\w\.ytimg\.com/vi/ibhaQZB9TWU/default\.jpg|, vs.thumb_url
|
37
|
+
assert_match %r|^<object\s+.*</object>$|, vs.embed_tag
|
38
|
+
assert_equal '水着の人妻', vs.title
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestYourFileHost < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
WWW::VideoScraper.configure do |conf|
|
9
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def teardown
|
14
|
+
# FileUtils.remove_entry_secure(@cache_root, true)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_scrape
|
18
|
+
vs = WWW::VideoScraper.scrape('http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv')
|
19
|
+
assert_equal 'http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv', vs.page_url
|
20
|
+
assert_match %r|http://cdn\w*\.yourfilehost\.com/unit1/flash8/\d+/[[:alnum:]]{32}\.flv|, vs.video_url
|
21
|
+
assert_match %r|http://cdn\w*\.yourfilehost\.com/thumbs/\d+/[[:alnum:]]{32}\.jpg|, vs.thumb_url
|
22
|
+
assert_equal 'XV436__03.wmv', vs.title
|
23
|
+
end
|
24
|
+
end
|