video_scraper 1.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/ChangeLog +4 -0
- data/README +71 -0
- data/Rakefile +146 -0
- data/lib/www/video_scraper.rb +88 -0
- data/lib/www/video_scraper/adult_satellites.rb +27 -0
- data/lib/www/video_scraper/age_sage.rb +28 -0
- data/lib/www/video_scraper/ameba_vision.rb +22 -0
- data/lib/www/video_scraper/base.rb +88 -0
- data/lib/www/video_scraper/dailymotion.rb +30 -0
- data/lib/www/video_scraper/eic_book.rb +34 -0
- data/lib/www/video_scraper/moro_tube.rb +31 -0
- data/lib/www/video_scraper/nico_video.rb +68 -0
- data/lib/www/video_scraper/pornhub.rb +24 -0
- data/lib/www/video_scraper/pornotube.rb +39 -0
- data/lib/www/video_scraper/red_tube.rb +89 -0
- data/lib/www/video_scraper/tube8.rb +31 -0
- data/lib/www/video_scraper/veoh.rb +28 -0
- data/lib/www/video_scraper/you_porn.rb +26 -0
- data/lib/www/video_scraper/you_tube.rb +53 -0
- data/lib/www/video_scraper/your_file_host.rb +54 -0
- data/test/test_helper.rb +23 -0
- data/test/www/test_video_scraper.rb +43 -0
- data/test/www/video_scraper/test_adult_satellites.rb +13 -0
- data/test/www/video_scraper/test_age_sage.rb +13 -0
- data/test/www/video_scraper/test_ameba_vision.rb +12 -0
- data/test/www/video_scraper/test_base.rb +14 -0
- data/test/www/video_scraper/test_dailymotion.rb +14 -0
- data/test/www/video_scraper/test_eic_book.rb +14 -0
- data/test/www/video_scraper/test_moro_tube.rb +13 -0
- data/test/www/video_scraper/test_nico_video.rb +23 -0
- data/test/www/video_scraper/test_pornhub.rb +14 -0
- data/test/www/video_scraper/test_pornotube.rb +21 -0
- data/test/www/video_scraper/test_red_tube.rb +13 -0
- data/test/www/video_scraper/test_tube8.rb +14 -0
- data/test/www/video_scraper/test_veoh.rb +24 -0
- data/test/www/video_scraper/test_you_porn.rb +13 -0
- data/test/www/video_scraper/test_you_tube.rb +32 -0
- data/test/www/video_scraper/test_your_file_host.rb +14 -0
- metadata +133 -0
@@ -0,0 +1,54 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.expand_path(File.dirname(__FILE__) + '/base')
|
4
|
+
|
5
|
+
module WWW
|
6
|
+
module VideoScraper
|
7
|
+
class YourFileHost < Base
|
8
|
+
url_regex %r!\Ahttp://www\.yourfilehost\.com/media\.php\?cat=video&file=.+$!
|
9
|
+
|
10
|
+
class MaximumVideoPlaysReached < TryAgainLater; end
|
11
|
+
class BandwidthAllowanceExceeded < TryAgainLater; end
|
12
|
+
class NoFileCategory < FileNotFound; end
|
13
|
+
|
14
|
+
def filename
|
15
|
+
uri = URI.parse(@page_url)
|
16
|
+
q = CGI.parse(uri.query)
|
17
|
+
q['file'][0]
|
18
|
+
end
|
19
|
+
alias :title :filename
|
20
|
+
|
21
|
+
def scrape
|
22
|
+
html = http_get(@page_url)
|
23
|
+
doc = Hpricot(html.toutf8)
|
24
|
+
if elem = doc.at('//object[@id="objectPlayer"] //param[@name="movie"]')
|
25
|
+
value = elem.attributes['value']
|
26
|
+
raise StandardError, 'video information is not found' unless value
|
27
|
+
v = CGI::parse(value)
|
28
|
+
if request_url = v['video'][0]
|
29
|
+
response_body = http_get(request_url)
|
30
|
+
q = CGI::parse(response_body)
|
31
|
+
@thumb_url = q['photo'][0] rescue ''
|
32
|
+
@video_url = q['video_id'][0] rescue ''
|
33
|
+
end
|
34
|
+
elsif elem = doc.at('//object[@id="VIDEO"] //param[@name="URL"]')
|
35
|
+
@video_url = elem.attributes['value']
|
36
|
+
else
|
37
|
+
if html =~ /MAXIMUM VIDEO PLAYS REACHED/i
|
38
|
+
raise MaximumVideoPlaysReached, 'MAXIMUM VIDEO PLAYS REACHED'
|
39
|
+
elsif html =~ /Bandwidth Allowance exceeded/i
|
40
|
+
raise BandwidthAllowanceExceeded, 'Bandwidth Allowance exceeded'
|
41
|
+
elsif html =~ /url=error\.php\?err=8/i
|
42
|
+
raise FileNotFound, 'file not found'
|
43
|
+
elsif html =~ /url=error\.php\?err=5/i or html =~ /no file category/i
|
44
|
+
raise NoFileCategory, 'no file category'
|
45
|
+
elsif html =~ /File not found/i
|
46
|
+
raise FileNotFound, 'file not found'
|
47
|
+
else
|
48
|
+
raise TryAgainLater, 'scrape failed'
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require 'test/unit'
|
4
|
+
require File.expand_path(File.dirname(__FILE__) + '/../lib/www/video_scraper')
|
5
|
+
|
6
|
+
require 'fileutils'
|
7
|
+
require 'filecache'
|
8
|
+
require 'logger'
|
9
|
+
require 'pit'
|
10
|
+
|
11
|
+
class Test::Unit::TestCase
|
12
|
+
def logger
|
13
|
+
@logger ||= Logger.new(STDOUT)
|
14
|
+
end
|
15
|
+
|
16
|
+
def filecache
|
17
|
+
@filecache ||= FileCache.new('TestVideoScraper', '/tmp/test_video_scraper_cache', 60 * 60)
|
18
|
+
end
|
19
|
+
|
20
|
+
def default_opt
|
21
|
+
@default_opt ||= { :logger => logger, :cache => filecache }
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../test_helper'
|
4
|
+
|
5
|
+
class TestVideoScraper < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
end
|
9
|
+
|
10
|
+
def teardown
|
11
|
+
FileUtils.remove_entry_secure(@cache_root, true)
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_configure
|
15
|
+
WWW::VideoScraper.options = {}
|
16
|
+
|
17
|
+
assert_nil WWW::VideoScraper.options[:cache]
|
18
|
+
assert_nil WWW::VideoScraper.options[:logger]
|
19
|
+
|
20
|
+
WWW::VideoScraper.configure do |conf|
|
21
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
22
|
+
conf[:logger] = Logger.new(STDOUT)
|
23
|
+
end
|
24
|
+
|
25
|
+
assert_kind_of FileCache, WWW::VideoScraper.options[:cache]
|
26
|
+
assert_kind_of Logger, WWW::VideoScraper.options[:logger]
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_find_module
|
30
|
+
mod = WWW::VideoScraper.find_module('http://jp.youtube.com/watch?v=Ym20IwIUbuU')
|
31
|
+
assert_equal WWW::VideoScraper::YouTube, mod
|
32
|
+
mod = WWW::VideoScraper.find_module('http://www.nicovideo.jp/watch/sm1175788')
|
33
|
+
assert_equal WWW::VideoScraper::NicoVideo, mod
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_scrape
|
37
|
+
mod = WWW::VideoScraper.find_module('http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv')
|
38
|
+
assert_equal WWW::VideoScraper::YourFileHost, mod
|
39
|
+
vs = WWW::VideoScraper.scrape('http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv')
|
40
|
+
assert_kind_of WWW::VideoScraper::YourFileHost, vs
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestAdultSatellites < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::AdultSatellites.scrape('http://www.asa.tv/movie_detail.php?userid=&movie_id=15680', default_opt)
|
8
|
+
assert_equal 'http://www.asa.tv/movie_detail.php?userid=&movie_id=15680', vs.page_url
|
9
|
+
assert_match %r|http://asa\.tv/movie/D5/gcuppapa/[[:alnum:]]{32}\.flv|, vs.video_url
|
10
|
+
assert_match %r|http://www\.asa\.tv/captured/gcuppapa/[[:alnum:]]{32}_1\.jpg|, vs.thumb_url
|
11
|
+
assert_equal '妃乃ひかり,2', vs.title
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestAgeSage < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::AgeSage.scrape('http://adult.agesage.jp/contentsPage.html?mcd=oadyChZgcoN9rqbJ', default_opt)
|
8
|
+
assert_equal 'http://adult.agesage.jp/contentsPage.html?mcd=oadyChZgcoN9rqbJ', vs.page_url
|
9
|
+
assert_match %r!http://file\d+\.agesage\.jp/flv/\d{8}/\d{10}_\d{6}\.flv!, vs.video_url
|
10
|
+
assert_match %r!http://file\d+\.agesage\.jp/data/\d{8}/\d{10}_\d{6}!, vs.thumb_url
|
11
|
+
assert_match %r!^<script type=\"text/javascript\" src=\".*</script>$!, vs.embed_tag
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestAmebaVision < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::AmebaVision.scrape('http://vision.ameba.jp/watch.do?movie=772341', default_opt)
|
8
|
+
assert_equal 'http://vision.ameba.jp/watch.do?movie=772341', vs.page_url
|
9
|
+
assert_match %r|http://[-[:alnum:]]+\.vision\.ameba\.jp/flv/\d{4}/\d{2}/\d{2}/[[:alnum:]]+\.flv|, vs.video_url
|
10
|
+
assert_match %r|http://[-[:alnum:]]+\.vision\.ameba\.jp/jpg/\d{4}/\d{2}/\d{2}/[[:alnum:]]+_4\.jpg|, vs.thumb_url
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestBase < Test::Unit::TestCase
|
6
|
+
def test_nulllogger
|
7
|
+
logger = WWW::VideoScraper::NullLogger.new
|
8
|
+
assert_nil logger.fatal 'nop'
|
9
|
+
assert_nil logger.error 'nop'
|
10
|
+
assert_nil logger.warn 'nop'
|
11
|
+
assert_nil logger.info 'nop'
|
12
|
+
assert_nil logger.debug 'nop'
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestDailymotion < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::Dailymotion.scrape('http://www.dailymotion.com/mokelov/japan/video/x12gr3_music', default_opt)
|
8
|
+
assert_equal 'http://www.dailymotion.com/mokelov/japan/video/x12gr3_music', vs.page_url
|
9
|
+
assert_match %r|http://www\.dailymotion\.com/get/\d{2}/\d+x\d+/flv/\d+\.flv\?key=[[:alnum:]]+|, vs.video_url
|
10
|
+
assert_match %r|http://\w+\.static\.dailymotion\.com/dyn/preview/\d+x\d+/\d+\.jpg\?\d{14}|, vs.thumb_url
|
11
|
+
assert_match %r|^<div><object\s.*</div>$|, vs.embed_tag
|
12
|
+
assert_equal '遭難 東京事変', vs.title
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class EicBook < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::EicBook.scrape('http://www.eic-book.com/detail_12759.html', default_opt)
|
8
|
+
assert_equal 'http://www.eic-book.com/detail_12759.html?flg=sm', vs.page_url
|
9
|
+
assert_equal 'http://flv.idol-mile.com/book/12759.flv', vs.video_url
|
10
|
+
assert_equal 'http://www.eic-book.com/img/product/h4/pp_12759.jpg', vs.thumb_url
|
11
|
+
assert_equal '藤木あやか DVD 「お蔵入り寸前!藤木あやか A面」', vs.title
|
12
|
+
assert_equal 24, vs.capture_urls.count
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestMoroTube < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::MoroTube.scrape('http://www.morotube.com/watch.php?clip=46430e1d', default_opt)
|
8
|
+
assert_equal 'http://www.morotube.com/watch.php?clip=46430e1d', vs.page_url
|
9
|
+
assert_match %r!http://video\d+\.morotube\.com/[[:alnum:]]{32}/[[:alnum:]]{8}/[[:alnum:]]{8}\.flv!, vs.video_url
|
10
|
+
assert_match %r!http://static\d+\.morotube\.com/thumbs/\w{8}\.jpg!, vs.thumb_url
|
11
|
+
assert_match %r!^<embed\s.*</embed>$!, vs.embed_tag
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestNicoVideo < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
config = Pit.get('nicovideo.jp', :require => {
|
8
|
+
'mail' => 'your email in nicovideo.jp',
|
9
|
+
'password' => 'your password in nicovideo.jp'
|
10
|
+
})
|
11
|
+
default_opt.merge!(:nico_video_mail => config['mail'],
|
12
|
+
:nico_video_password => config['password'])
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_scrape
|
16
|
+
vs = WWW::VideoScraper::NicoVideo.scrape('http://www.nicovideo.jp/watch/sm1175788', default_opt)
|
17
|
+
assert_equal 'http://www.nicovideo.jp/watch/sm1175788', vs.page_url
|
18
|
+
assert_match %r|http://smile-[[:alnum:]]+\.nicovideo\.jp/smile\?v=\d{7}\.\d{5}|, vs.video_url
|
19
|
+
assert_match %r|http://tn-skr\d+\.smilevideo\.jp/smile\?i=\d{7}|, vs.thumb_url
|
20
|
+
assert_match %r|^<iframe\s+.*</iframe>$|, vs.embed_tag
|
21
|
+
assert_equal '本格的 ガチムチパンツレスリング', vs.title
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestPornhub < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::Pornhub.scrape('http://www.pornhub.com/view_video.php?viewkey=27f115e7fee8c18f92b0', default_opt)
|
8
|
+
assert_equal 'http://www.pornhub.com/view_video.php?viewkey=27f115e7fee8c18f92b0', vs.page_url
|
9
|
+
assert_match %r|http://media1.pornhub.com/dl/[[:alnum:]]{32}/[[:alnum:]]{8}/videos/000/191/743/191743\.flv|, vs.video_url
|
10
|
+
assert_equal 'http://p1.pornhub.com/thumbs/000/191/743/small.jpg', vs.thumb_url
|
11
|
+
assert_match %r|^<object type=\"application/x-shockwave-flash\" data=\".*</object>$|, vs.embed_tag
|
12
|
+
assert_equal 'Liliane Tiger and Jane Darling to hot to handle', vs.title
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestPornotube < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::Pornotube.scrape('http://pornotube.com/media.php?m=1677937', default_opt)
|
8
|
+
assert_equal 'http://pornotube.com/media.php?m=1677937', vs.page_url
|
9
|
+
assert_match %r|http://video\d+\.pornotube\.com/\d+/\d+\.flv|, vs.video_url
|
10
|
+
assert_match %r|http://photo\.pornotube\.com/thumbnails/video/\d+/\d+\.jpg|, vs.thumb_url
|
11
|
+
assert_match %r|^<embed\s+src=\".*\"\s*/>$|, vs.embed_tag
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_scrape_alt_url
|
15
|
+
vs = WWW::VideoScraper::Pornotube.scrape('http://pornotube.com/channels.php?channelId=83&m=1677912', default_opt)
|
16
|
+
assert_equal 'http://pornotube.com/channels.php?channelId=83&m=1677912', vs.page_url
|
17
|
+
assert_match %r|http://video\d+\.pornotube\.com/\d+/\d+\.flv|, vs.video_url
|
18
|
+
assert_match %r|http://photo\.pornotube\.com/thumbnails/video/\d+/\d+\.jpg|, vs.thumb_url
|
19
|
+
assert_match %r|^<embed\s+src=\".*\"\s*/>$|, vs.embed_tag
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestRedTube < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::RedTube.scrape('http://www.redtube.com/15021', default_opt)
|
8
|
+
assert_equal 'http://www.redtube.com/15021', vs.page_url
|
9
|
+
assert_equal 'http://dl.redtube.com/_videos_t4vn23s9jc5498tgj49icfj4678/0000015/X6KKXB0DB.flv', vs.video_url
|
10
|
+
assert_equal 'http://thumbs.redtube.com/_thumbs/0000015/0015021/0015021_001.jpg', vs.thumb_url
|
11
|
+
assert_match %r!<object\s+.*</object>!, vs.embed_tag
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestTube8 < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::Tube8.scrape('http://www.tube8.com/anal/alexis-amore-pov/56983/', default_opt)
|
8
|
+
assert_equal 'http://www.tube8.com/anal/alexis-amore-pov/56983/', vs.page_url
|
9
|
+
assert_match %r|http://media\w+\.tube8\.com/flv/[[:alnum:]]{32}/[[:alnum:]]{8}/\d{4}/\d{2}/[[:alnum:]]+/[[:alnum:]]+\.flv|, vs.video_url
|
10
|
+
assert_equal 'http://www.tube8.com/vs/83/56983.jpg', vs.thumb_url
|
11
|
+
assert_equal 'Alexis Amore POV', vs.title
|
12
|
+
assert_match %r!http://mobile\d+\.tube8\.com/flv/[[:alnum:]]{32}/[[:alnum:]]{8}/\d{4}/\d{2}/[[:alnum:]]+/[[:alnum:]]+\.3gp!, vs.video_url_3gp
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestVeoh < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::Veoh.scrape('http://www.veoh.com/videos/v6245232rh8aGEM9', default_opt)
|
8
|
+
assert_equal 'http://www.veoh.com/videos/v6245232rh8aGEM9', vs.page_url
|
9
|
+
assert_match %r|http://content\.veoh\.com/flash/p/\d/[[:alnum:]]{16}/[[:alnum:]]{40}\.fll\?ct=[[:alnum:]]{48}|, vs.video_url
|
10
|
+
assert_match %r|http://p-images\.veoh\.com/image\.out\?imageId=media-[[:alnum:]]+.jpg|, vs.thumb_url
|
11
|
+
assert_match %r|^<object\s.*>$|, vs.embed_tag
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_canonical_url
|
15
|
+
vs = WWW::VideoScraper::Veoh.scrape('http://www.veoh.com/collection/maysaku/watch/v19937773gwSJPMk', default_opt)
|
16
|
+
assert_equal 'http://www.veoh.com/videos/v19937773gwSJPMk', vs.page_url
|
17
|
+
vs = WWW::VideoScraper::Veoh.scrape('http://www.veoh.com/collection/maysaku/watch/v19937773gwSJPMk#watch%3Dv16112008KGD7Pg2n', default_opt)
|
18
|
+
assert_equal 'http://www.veoh.com/videos/v16112008KGD7Pg2n', vs.page_url
|
19
|
+
vs = WWW::VideoScraper::Veoh.scrape('http://www.veoh.com/videos/v19937773gwSJPMk?rank=0&jsonParams=%7B%22numResults%22%3A20%2C%22rlmin%22%3A0%2C%22query%22%3A%22Shaman+King+01%22%2C%22rlmax%22%3Anull%2C%22veohOnly%22%3Atrue%2C%22order%22%3A%22default%22%2C%22range%22%3A%22a%22%2C%22sId%22%3A%22192998624295114150%22%7D&searchId=192998624295114150&rank=1', default_opt)
|
20
|
+
assert_equal 'http://www.veoh.com/videos/v19937773gwSJPMk', vs.page_url
|
21
|
+
vs = WWW::VideoScraper::Veoh.scrape('http://www.veoh.com/browse/videos/category/comedy/watch/v17078605sszQzbBF')
|
22
|
+
assert_equal 'http://www.veoh.com/videos/v17078605sszQzbBF', vs.page_url
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestYouPorn < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::YouPorn.scrape('http://youporn.com/watch/93495?user_choice=Enter', default_opt)
|
8
|
+
assert_equal 'http://youporn.com/watch/93495?user_choice=Enter', vs.page_url
|
9
|
+
assert_match %r|http://download\.youporn\.com/download/\d+/flv/\d+_.*\.flv.*|, vs.video_url
|
10
|
+
assert_match %r|http://ss-\d+\.youporn\.com/screenshot/\d+/\d+/screenshot/\d+_large\.jpg|, vs.thumb_url
|
11
|
+
assert_equal 'unbelievable Handjob!', vs.title
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestYouTube < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
config = Pit.get('youtube.com', :require => {
|
8
|
+
'username' => 'your email in youtube.com',
|
9
|
+
'password' => 'your password in youtube.com'
|
10
|
+
})
|
11
|
+
default_opt.merge!(:you_tube_username => config['username'],
|
12
|
+
:you_tube_password => config['password'])
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_scrape
|
16
|
+
vs = WWW::VideoScraper::YouTube.scrape('http://jp.youtube.com/watch?v=Ym20IwIUbuU', default_opt)
|
17
|
+
assert_equal 'http://jp.youtube.com/watch?v=Ym20IwIUbuU', vs.page_url
|
18
|
+
assert_match %r|http://jp\.youtube\.com/get_video\?video_id=Ym20IwIUbuU&t=[-_[:alnum:]]{32}|, vs.video_url
|
19
|
+
assert_match %r|http://\w\.ytimg\.com/vi/Ym20IwIUbuU/default\.jpg|, vs.thumb_url
|
20
|
+
assert_match %r|^<object\s+.*</object>$|, vs.embed_tag
|
21
|
+
assert_equal 'とらドラ!Toradora ep15 2-3', vs.title
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_scrape_alt_url
|
25
|
+
vs = WWW::VideoScraper::YouTube.scrape('http://jp.youtube.com/watch?v=ATHC8qDkoO0', default_opt)
|
26
|
+
assert_equal 'http://jp.youtube.com/watch?v=ATHC8qDkoO0', vs.page_url
|
27
|
+
assert_match %r|http://jp\.youtube\.com/get_video\?video_id=ATHC8qDkoO0&t=[-_[:alnum:]]{32}|, vs.video_url
|
28
|
+
assert_match %r|http://\w\.ytimg\.com/vi/ATHC8qDkoO0/default\.jpg|, vs.thumb_url
|
29
|
+
assert_match %r|^<object\s+.*</object>$|, vs.embed_tag
|
30
|
+
assert_equal "019' Sexii_Shower 藤澤まお_洗白白", vs.title
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestYourFileHost < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::YourFileHost.scrape('http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv', default_opt)
|
8
|
+
assert_equal 'http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv', vs.page_url
|
9
|
+
assert_match %r|http://cdn\w*\.yourfilehost\.com/unit1/flash8/\d+/[[:alnum:]]{32}\.flv|, vs.video_url
|
10
|
+
assert_match %r|http://cdn\w*\.yourfilehost\.com/thumbs/\d+/[[:alnum:]]{32}\.jpg|, vs.thumb_url
|
11
|
+
assert_equal 'XV436__03.wmv', vs.title
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
metadata
ADDED
@@ -0,0 +1,133 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: video_scraper
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.5
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- YAMAGUCHI Seiji
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-10-23 00:00:00 +09:00
|
13
|
+
default_executable:
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: mechanize
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 0.8.4
|
24
|
+
version:
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: hpricot
|
27
|
+
type: :runtime
|
28
|
+
version_requirement:
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.6.164
|
34
|
+
version:
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: json
|
37
|
+
type: :runtime
|
38
|
+
version_requirement:
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 1.1.3
|
44
|
+
version:
|
45
|
+
description: Web scraping library for video sharing sites.
|
46
|
+
email: valda@underscore.jp
|
47
|
+
executables: []
|
48
|
+
|
49
|
+
extensions: []
|
50
|
+
|
51
|
+
extra_rdoc_files:
|
52
|
+
- README
|
53
|
+
- ChangeLog
|
54
|
+
files:
|
55
|
+
- README
|
56
|
+
- ChangeLog
|
57
|
+
- Rakefile
|
58
|
+
- test/www/video_scraper/test_adult_satellites.rb
|
59
|
+
- test/www/video_scraper/test_base.rb
|
60
|
+
- test/www/video_scraper/test_pornotube.rb
|
61
|
+
- test/www/video_scraper/test_moro_tube.rb
|
62
|
+
- test/www/video_scraper/test_tube8.rb
|
63
|
+
- test/www/video_scraper/test_pornhub.rb
|
64
|
+
- test/www/video_scraper/test_your_file_host.rb
|
65
|
+
- test/www/video_scraper/test_nico_video.rb
|
66
|
+
- test/www/video_scraper/test_eic_book.rb
|
67
|
+
- test/www/video_scraper/test_you_porn.rb
|
68
|
+
- test/www/video_scraper/test_dailymotion.rb
|
69
|
+
- test/www/video_scraper/test_ameba_vision.rb
|
70
|
+
- test/www/video_scraper/test_age_sage.rb
|
71
|
+
- test/www/video_scraper/test_veoh.rb
|
72
|
+
- test/www/video_scraper/test_you_tube.rb
|
73
|
+
- test/www/video_scraper/test_red_tube.rb
|
74
|
+
- test/www/test_video_scraper.rb
|
75
|
+
- test/test_helper.rb
|
76
|
+
- lib/www/video_scraper/you_tube.rb
|
77
|
+
- lib/www/video_scraper/your_file_host.rb
|
78
|
+
- lib/www/video_scraper/red_tube.rb
|
79
|
+
- lib/www/video_scraper/dailymotion.rb
|
80
|
+
- lib/www/video_scraper/base.rb
|
81
|
+
- lib/www/video_scraper/you_porn.rb
|
82
|
+
- lib/www/video_scraper/moro_tube.rb
|
83
|
+
- lib/www/video_scraper/veoh.rb
|
84
|
+
- lib/www/video_scraper/eic_book.rb
|
85
|
+
- lib/www/video_scraper/nico_video.rb
|
86
|
+
- lib/www/video_scraper/ameba_vision.rb
|
87
|
+
- lib/www/video_scraper/adult_satellites.rb
|
88
|
+
- lib/www/video_scraper/pornotube.rb
|
89
|
+
- lib/www/video_scraper/tube8.rb
|
90
|
+
- lib/www/video_scraper/pornhub.rb
|
91
|
+
- lib/www/video_scraper/age_sage.rb
|
92
|
+
- lib/www/video_scraper.rb
|
93
|
+
has_rdoc: true
|
94
|
+
homepage: http://github.com/valda/video_scraper
|
95
|
+
licenses: []
|
96
|
+
|
97
|
+
post_install_message:
|
98
|
+
rdoc_options:
|
99
|
+
- --title
|
100
|
+
- video_scraper documentation
|
101
|
+
- --charset
|
102
|
+
- utf-8
|
103
|
+
- --opname
|
104
|
+
- index.html
|
105
|
+
- --line-numbers
|
106
|
+
- --main
|
107
|
+
- README
|
108
|
+
- --inline-source
|
109
|
+
- --exclude
|
110
|
+
- ^(examples|extras)/
|
111
|
+
require_paths:
|
112
|
+
- lib
|
113
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: "0"
|
118
|
+
version:
|
119
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - ">="
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: "0"
|
124
|
+
version:
|
125
|
+
requirements: []
|
126
|
+
|
127
|
+
rubyforge_project: video_scraper
|
128
|
+
rubygems_version: 1.3.5
|
129
|
+
signing_key:
|
130
|
+
specification_version: 3
|
131
|
+
summary: Web scraping library for video sharing sites.
|
132
|
+
test_files: []
|
133
|
+
|