video_scraper 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/ChangeLog +4 -0
- data/README +71 -0
- data/Rakefile +146 -0
- data/lib/www/video_scraper.rb +88 -0
- data/lib/www/video_scraper/adult_satellites.rb +27 -0
- data/lib/www/video_scraper/age_sage.rb +28 -0
- data/lib/www/video_scraper/ameba_vision.rb +22 -0
- data/lib/www/video_scraper/base.rb +88 -0
- data/lib/www/video_scraper/dailymotion.rb +30 -0
- data/lib/www/video_scraper/eic_book.rb +34 -0
- data/lib/www/video_scraper/moro_tube.rb +31 -0
- data/lib/www/video_scraper/nico_video.rb +68 -0
- data/lib/www/video_scraper/pornhub.rb +24 -0
- data/lib/www/video_scraper/pornotube.rb +39 -0
- data/lib/www/video_scraper/red_tube.rb +89 -0
- data/lib/www/video_scraper/tube8.rb +31 -0
- data/lib/www/video_scraper/veoh.rb +28 -0
- data/lib/www/video_scraper/you_porn.rb +26 -0
- data/lib/www/video_scraper/you_tube.rb +53 -0
- data/lib/www/video_scraper/your_file_host.rb +54 -0
- data/test/test_helper.rb +23 -0
- data/test/www/test_video_scraper.rb +43 -0
- data/test/www/video_scraper/test_adult_satellites.rb +13 -0
- data/test/www/video_scraper/test_age_sage.rb +13 -0
- data/test/www/video_scraper/test_ameba_vision.rb +12 -0
- data/test/www/video_scraper/test_base.rb +14 -0
- data/test/www/video_scraper/test_dailymotion.rb +14 -0
- data/test/www/video_scraper/test_eic_book.rb +14 -0
- data/test/www/video_scraper/test_moro_tube.rb +13 -0
- data/test/www/video_scraper/test_nico_video.rb +23 -0
- data/test/www/video_scraper/test_pornhub.rb +14 -0
- data/test/www/video_scraper/test_pornotube.rb +21 -0
- data/test/www/video_scraper/test_red_tube.rb +13 -0
- data/test/www/video_scraper/test_tube8.rb +14 -0
- data/test/www/video_scraper/test_veoh.rb +24 -0
- data/test/www/video_scraper/test_you_porn.rb +13 -0
- data/test/www/video_scraper/test_you_tube.rb +32 -0
- data/test/www/video_scraper/test_your_file_host.rb +14 -0
- metadata +133 -0
@@ -0,0 +1,54 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.expand_path(File.dirname(__FILE__) + '/base')
|
4
|
+
|
5
|
+
module WWW
|
6
|
+
module VideoScraper
|
7
|
+
class YourFileHost < Base
|
8
|
+
url_regex %r!\Ahttp://www\.yourfilehost\.com/media\.php\?cat=video&file=.+$!
|
9
|
+
|
10
|
+
class MaximumVideoPlaysReached < TryAgainLater; end
|
11
|
+
class BandwidthAllowanceExceeded < TryAgainLater; end
|
12
|
+
class NoFileCategory < FileNotFound; end
|
13
|
+
|
14
|
+
def filename
|
15
|
+
uri = URI.parse(@page_url)
|
16
|
+
q = CGI.parse(uri.query)
|
17
|
+
q['file'][0]
|
18
|
+
end
|
19
|
+
alias :title :filename
|
20
|
+
|
21
|
+
def scrape
|
22
|
+
html = http_get(@page_url)
|
23
|
+
doc = Hpricot(html.toutf8)
|
24
|
+
if elem = doc.at('//object[@id="objectPlayer"] //param[@name="movie"]')
|
25
|
+
value = elem.attributes['value']
|
26
|
+
raise StandardError, 'video information is not found' unless value
|
27
|
+
v = CGI::parse(value)
|
28
|
+
if request_url = v['video'][0]
|
29
|
+
response_body = http_get(request_url)
|
30
|
+
q = CGI::parse(response_body)
|
31
|
+
@thumb_url = q['photo'][0] rescue ''
|
32
|
+
@video_url = q['video_id'][0] rescue ''
|
33
|
+
end
|
34
|
+
elsif elem = doc.at('//object[@id="VIDEO"] //param[@name="URL"]')
|
35
|
+
@video_url = elem.attributes['value']
|
36
|
+
else
|
37
|
+
if html =~ /MAXIMUM VIDEO PLAYS REACHED/i
|
38
|
+
raise MaximumVideoPlaysReached, 'MAXIMUM VIDEO PLAYS REACHED'
|
39
|
+
elsif html =~ /Bandwidth Allowance exceeded/i
|
40
|
+
raise BandwidthAllowanceExceeded, 'Bandwidth Allowance exceeded'
|
41
|
+
elsif html =~ /url=error\.php\?err=8/i
|
42
|
+
raise FileNotFound, 'file not found'
|
43
|
+
elsif html =~ /url=error\.php\?err=5/i or html =~ /no file category/i
|
44
|
+
raise NoFileCategory, 'no file category'
|
45
|
+
elsif html =~ /File not found/i
|
46
|
+
raise FileNotFound, 'file not found'
|
47
|
+
else
|
48
|
+
raise TryAgainLater, 'scrape failed'
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require 'test/unit'
|
4
|
+
require File.expand_path(File.dirname(__FILE__) + '/../lib/www/video_scraper')
|
5
|
+
|
6
|
+
require 'fileutils'
|
7
|
+
require 'filecache'
|
8
|
+
require 'logger'
|
9
|
+
require 'pit'
|
10
|
+
|
11
|
+
class Test::Unit::TestCase
|
12
|
+
def logger
|
13
|
+
@logger ||= Logger.new(STDOUT)
|
14
|
+
end
|
15
|
+
|
16
|
+
def filecache
|
17
|
+
@filecache ||= FileCache.new('TestVideoScraper', '/tmp/test_video_scraper_cache', 60 * 60)
|
18
|
+
end
|
19
|
+
|
20
|
+
def default_opt
|
21
|
+
@default_opt ||= { :logger => logger, :cache => filecache }
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../test_helper'
|
4
|
+
|
5
|
+
class TestVideoScraper < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@cache_root = '/tmp/test_video_scraper_cache'
|
8
|
+
end
|
9
|
+
|
10
|
+
def teardown
|
11
|
+
FileUtils.remove_entry_secure(@cache_root, true)
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_configure
|
15
|
+
WWW::VideoScraper.options = {}
|
16
|
+
|
17
|
+
assert_nil WWW::VideoScraper.options[:cache]
|
18
|
+
assert_nil WWW::VideoScraper.options[:logger]
|
19
|
+
|
20
|
+
WWW::VideoScraper.configure do |conf|
|
21
|
+
conf[:cache] = FileCache.new('TestVideoScraper', @cache_root, 60*60*24)
|
22
|
+
conf[:logger] = Logger.new(STDOUT)
|
23
|
+
end
|
24
|
+
|
25
|
+
assert_kind_of FileCache, WWW::VideoScraper.options[:cache]
|
26
|
+
assert_kind_of Logger, WWW::VideoScraper.options[:logger]
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_find_module
|
30
|
+
mod = WWW::VideoScraper.find_module('http://jp.youtube.com/watch?v=Ym20IwIUbuU')
|
31
|
+
assert_equal WWW::VideoScraper::YouTube, mod
|
32
|
+
mod = WWW::VideoScraper.find_module('http://www.nicovideo.jp/watch/sm1175788')
|
33
|
+
assert_equal WWW::VideoScraper::NicoVideo, mod
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_scrape
|
37
|
+
mod = WWW::VideoScraper.find_module('http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv')
|
38
|
+
assert_equal WWW::VideoScraper::YourFileHost, mod
|
39
|
+
vs = WWW::VideoScraper.scrape('http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv')
|
40
|
+
assert_kind_of WWW::VideoScraper::YourFileHost, vs
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestAdultSatellites < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::AdultSatellites.scrape('http://www.asa.tv/movie_detail.php?userid=&movie_id=15680', default_opt)
|
8
|
+
assert_equal 'http://www.asa.tv/movie_detail.php?userid=&movie_id=15680', vs.page_url
|
9
|
+
assert_match %r|http://asa\.tv/movie/D5/gcuppapa/[[:alnum:]]{32}\.flv|, vs.video_url
|
10
|
+
assert_match %r|http://www\.asa\.tv/captured/gcuppapa/[[:alnum:]]{32}_1\.jpg|, vs.thumb_url
|
11
|
+
assert_equal '妃乃ひかり,2', vs.title
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestAgeSage < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::AgeSage.scrape('http://adult.agesage.jp/contentsPage.html?mcd=oadyChZgcoN9rqbJ', default_opt)
|
8
|
+
assert_equal 'http://adult.agesage.jp/contentsPage.html?mcd=oadyChZgcoN9rqbJ', vs.page_url
|
9
|
+
assert_match %r!http://file\d+\.agesage\.jp/flv/\d{8}/\d{10}_\d{6}\.flv!, vs.video_url
|
10
|
+
assert_match %r!http://file\d+\.agesage\.jp/data/\d{8}/\d{10}_\d{6}!, vs.thumb_url
|
11
|
+
assert_match %r!^<script type=\"text/javascript\" src=\".*</script>$!, vs.embed_tag
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestAmebaVision < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::AmebaVision.scrape('http://vision.ameba.jp/watch.do?movie=772341', default_opt)
|
8
|
+
assert_equal 'http://vision.ameba.jp/watch.do?movie=772341', vs.page_url
|
9
|
+
assert_match %r|http://[-[:alnum:]]+\.vision\.ameba\.jp/flv/\d{4}/\d{2}/\d{2}/[[:alnum:]]+\.flv|, vs.video_url
|
10
|
+
assert_match %r|http://[-[:alnum:]]+\.vision\.ameba\.jp/jpg/\d{4}/\d{2}/\d{2}/[[:alnum:]]+_4\.jpg|, vs.thumb_url
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestBase < Test::Unit::TestCase
|
6
|
+
def test_nulllogger
|
7
|
+
logger = WWW::VideoScraper::NullLogger.new
|
8
|
+
assert_nil logger.fatal 'nop'
|
9
|
+
assert_nil logger.error 'nop'
|
10
|
+
assert_nil logger.warn 'nop'
|
11
|
+
assert_nil logger.info 'nop'
|
12
|
+
assert_nil logger.debug 'nop'
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestDailymotion < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::Dailymotion.scrape('http://www.dailymotion.com/mokelov/japan/video/x12gr3_music', default_opt)
|
8
|
+
assert_equal 'http://www.dailymotion.com/mokelov/japan/video/x12gr3_music', vs.page_url
|
9
|
+
assert_match %r|http://www\.dailymotion\.com/get/\d{2}/\d+x\d+/flv/\d+\.flv\?key=[[:alnum:]]+|, vs.video_url
|
10
|
+
assert_match %r|http://\w+\.static\.dailymotion\.com/dyn/preview/\d+x\d+/\d+\.jpg\?\d{14}|, vs.thumb_url
|
11
|
+
assert_match %r|^<div><object\s.*</div>$|, vs.embed_tag
|
12
|
+
assert_equal '遭難 東京事変', vs.title
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class EicBook < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::EicBook.scrape('http://www.eic-book.com/detail_12759.html', default_opt)
|
8
|
+
assert_equal 'http://www.eic-book.com/detail_12759.html?flg=sm', vs.page_url
|
9
|
+
assert_equal 'http://flv.idol-mile.com/book/12759.flv', vs.video_url
|
10
|
+
assert_equal 'http://www.eic-book.com/img/product/h4/pp_12759.jpg', vs.thumb_url
|
11
|
+
assert_equal '藤木あやか DVD 「お蔵入り寸前!藤木あやか A面」', vs.title
|
12
|
+
assert_equal 24, vs.capture_urls.count
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestMoroTube < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::MoroTube.scrape('http://www.morotube.com/watch.php?clip=46430e1d', default_opt)
|
8
|
+
assert_equal 'http://www.morotube.com/watch.php?clip=46430e1d', vs.page_url
|
9
|
+
assert_match %r!http://video\d+\.morotube\.com/[[:alnum:]]{32}/[[:alnum:]]{8}/[[:alnum:]]{8}\.flv!, vs.video_url
|
10
|
+
assert_match %r!http://static\d+\.morotube\.com/thumbs/\w{8}\.jpg!, vs.thumb_url
|
11
|
+
assert_match %r!^<embed\s.*</embed>$!, vs.embed_tag
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestNicoVideo < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
config = Pit.get('nicovideo.jp', :require => {
|
8
|
+
'mail' => 'your email in nicovideo.jp',
|
9
|
+
'password' => 'your password in nicovideo.jp'
|
10
|
+
})
|
11
|
+
default_opt.merge!(:nico_video_mail => config['mail'],
|
12
|
+
:nico_video_password => config['password'])
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_scrape
|
16
|
+
vs = WWW::VideoScraper::NicoVideo.scrape('http://www.nicovideo.jp/watch/sm1175788', default_opt)
|
17
|
+
assert_equal 'http://www.nicovideo.jp/watch/sm1175788', vs.page_url
|
18
|
+
assert_match %r|http://smile-[[:alnum:]]+\.nicovideo\.jp/smile\?v=\d{7}\.\d{5}|, vs.video_url
|
19
|
+
assert_match %r|http://tn-skr\d+\.smilevideo\.jp/smile\?i=\d{7}|, vs.thumb_url
|
20
|
+
assert_match %r|^<iframe\s+.*</iframe>$|, vs.embed_tag
|
21
|
+
assert_equal '本格的 ガチムチパンツレスリング', vs.title
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestPornhub < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::Pornhub.scrape('http://www.pornhub.com/view_video.php?viewkey=27f115e7fee8c18f92b0', default_opt)
|
8
|
+
assert_equal 'http://www.pornhub.com/view_video.php?viewkey=27f115e7fee8c18f92b0', vs.page_url
|
9
|
+
assert_match %r|http://media1.pornhub.com/dl/[[:alnum:]]{32}/[[:alnum:]]{8}/videos/000/191/743/191743\.flv|, vs.video_url
|
10
|
+
assert_equal 'http://p1.pornhub.com/thumbs/000/191/743/small.jpg', vs.thumb_url
|
11
|
+
assert_match %r|^<object type=\"application/x-shockwave-flash\" data=\".*</object>$|, vs.embed_tag
|
12
|
+
assert_equal 'Liliane Tiger and Jane Darling to hot to handle', vs.title
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestPornotube < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::Pornotube.scrape('http://pornotube.com/media.php?m=1677937', default_opt)
|
8
|
+
assert_equal 'http://pornotube.com/media.php?m=1677937', vs.page_url
|
9
|
+
assert_match %r|http://video\d+\.pornotube\.com/\d+/\d+\.flv|, vs.video_url
|
10
|
+
assert_match %r|http://photo\.pornotube\.com/thumbnails/video/\d+/\d+\.jpg|, vs.thumb_url
|
11
|
+
assert_match %r|^<embed\s+src=\".*\"\s*/>$|, vs.embed_tag
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_scrape_alt_url
|
15
|
+
vs = WWW::VideoScraper::Pornotube.scrape('http://pornotube.com/channels.php?channelId=83&m=1677912', default_opt)
|
16
|
+
assert_equal 'http://pornotube.com/channels.php?channelId=83&m=1677912', vs.page_url
|
17
|
+
assert_match %r|http://video\d+\.pornotube\.com/\d+/\d+\.flv|, vs.video_url
|
18
|
+
assert_match %r|http://photo\.pornotube\.com/thumbnails/video/\d+/\d+\.jpg|, vs.thumb_url
|
19
|
+
assert_match %r|^<embed\s+src=\".*\"\s*/>$|, vs.embed_tag
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestRedTube < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::RedTube.scrape('http://www.redtube.com/15021', default_opt)
|
8
|
+
assert_equal 'http://www.redtube.com/15021', vs.page_url
|
9
|
+
assert_equal 'http://dl.redtube.com/_videos_t4vn23s9jc5498tgj49icfj4678/0000015/X6KKXB0DB.flv', vs.video_url
|
10
|
+
assert_equal 'http://thumbs.redtube.com/_thumbs/0000015/0015021/0015021_001.jpg', vs.thumb_url
|
11
|
+
assert_match %r!<object\s+.*</object>!, vs.embed_tag
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestTube8 < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::Tube8.scrape('http://www.tube8.com/anal/alexis-amore-pov/56983/', default_opt)
|
8
|
+
assert_equal 'http://www.tube8.com/anal/alexis-amore-pov/56983/', vs.page_url
|
9
|
+
assert_match %r|http://media\w+\.tube8\.com/flv/[[:alnum:]]{32}/[[:alnum:]]{8}/\d{4}/\d{2}/[[:alnum:]]+/[[:alnum:]]+\.flv|, vs.video_url
|
10
|
+
assert_equal 'http://www.tube8.com/vs/83/56983.jpg', vs.thumb_url
|
11
|
+
assert_equal 'Alexis Amore POV', vs.title
|
12
|
+
assert_match %r!http://mobile\d+\.tube8\.com/flv/[[:alnum:]]{32}/[[:alnum:]]{8}/\d{4}/\d{2}/[[:alnum:]]+/[[:alnum:]]+\.3gp!, vs.video_url_3gp
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestVeoh < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::Veoh.scrape('http://www.veoh.com/videos/v6245232rh8aGEM9', default_opt)
|
8
|
+
assert_equal 'http://www.veoh.com/videos/v6245232rh8aGEM9', vs.page_url
|
9
|
+
assert_match %r|http://content\.veoh\.com/flash/p/\d/[[:alnum:]]{16}/[[:alnum:]]{40}\.fll\?ct=[[:alnum:]]{48}|, vs.video_url
|
10
|
+
assert_match %r|http://p-images\.veoh\.com/image\.out\?imageId=media-[[:alnum:]]+.jpg|, vs.thumb_url
|
11
|
+
assert_match %r|^<object\s.*>$|, vs.embed_tag
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_canonical_url
|
15
|
+
vs = WWW::VideoScraper::Veoh.scrape('http://www.veoh.com/collection/maysaku/watch/v19937773gwSJPMk', default_opt)
|
16
|
+
assert_equal 'http://www.veoh.com/videos/v19937773gwSJPMk', vs.page_url
|
17
|
+
vs = WWW::VideoScraper::Veoh.scrape('http://www.veoh.com/collection/maysaku/watch/v19937773gwSJPMk#watch%3Dv16112008KGD7Pg2n', default_opt)
|
18
|
+
assert_equal 'http://www.veoh.com/videos/v16112008KGD7Pg2n', vs.page_url
|
19
|
+
vs = WWW::VideoScraper::Veoh.scrape('http://www.veoh.com/videos/v19937773gwSJPMk?rank=0&jsonParams=%7B%22numResults%22%3A20%2C%22rlmin%22%3A0%2C%22query%22%3A%22Shaman+King+01%22%2C%22rlmax%22%3Anull%2C%22veohOnly%22%3Atrue%2C%22order%22%3A%22default%22%2C%22range%22%3A%22a%22%2C%22sId%22%3A%22192998624295114150%22%7D&searchId=192998624295114150&rank=1', default_opt)
|
20
|
+
assert_equal 'http://www.veoh.com/videos/v19937773gwSJPMk', vs.page_url
|
21
|
+
vs = WWW::VideoScraper::Veoh.scrape('http://www.veoh.com/browse/videos/category/comedy/watch/v17078605sszQzbBF')
|
22
|
+
assert_equal 'http://www.veoh.com/videos/v17078605sszQzbBF', vs.page_url
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestYouPorn < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::YouPorn.scrape('http://youporn.com/watch/93495?user_choice=Enter', default_opt)
|
8
|
+
assert_equal 'http://youporn.com/watch/93495?user_choice=Enter', vs.page_url
|
9
|
+
assert_match %r|http://download\.youporn\.com/download/\d+/flv/\d+_.*\.flv.*|, vs.video_url
|
10
|
+
assert_match %r|http://ss-\d+\.youporn\.com/screenshot/\d+/\d+/screenshot/\d+_large\.jpg|, vs.thumb_url
|
11
|
+
assert_equal 'unbelievable Handjob!', vs.title
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestYouTube < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
config = Pit.get('youtube.com', :require => {
|
8
|
+
'username' => 'your email in youtube.com',
|
9
|
+
'password' => 'your password in youtube.com'
|
10
|
+
})
|
11
|
+
default_opt.merge!(:you_tube_username => config['username'],
|
12
|
+
:you_tube_password => config['password'])
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_scrape
|
16
|
+
vs = WWW::VideoScraper::YouTube.scrape('http://jp.youtube.com/watch?v=Ym20IwIUbuU', default_opt)
|
17
|
+
assert_equal 'http://jp.youtube.com/watch?v=Ym20IwIUbuU', vs.page_url
|
18
|
+
assert_match %r|http://jp\.youtube\.com/get_video\?video_id=Ym20IwIUbuU&t=[-_[:alnum:]]{32}|, vs.video_url
|
19
|
+
assert_match %r|http://\w\.ytimg\.com/vi/Ym20IwIUbuU/default\.jpg|, vs.thumb_url
|
20
|
+
assert_match %r|^<object\s+.*</object>$|, vs.embed_tag
|
21
|
+
assert_equal 'とらドラ!Toradora ep15 2-3', vs.title
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_scrape_alt_url
|
25
|
+
vs = WWW::VideoScraper::YouTube.scrape('http://jp.youtube.com/watch?v=ATHC8qDkoO0', default_opt)
|
26
|
+
assert_equal 'http://jp.youtube.com/watch?v=ATHC8qDkoO0', vs.page_url
|
27
|
+
assert_match %r|http://jp\.youtube\.com/get_video\?video_id=ATHC8qDkoO0&t=[-_[:alnum:]]{32}|, vs.video_url
|
28
|
+
assert_match %r|http://\w\.ytimg\.com/vi/ATHC8qDkoO0/default\.jpg|, vs.thumb_url
|
29
|
+
assert_match %r|^<object\s+.*</object>$|, vs.embed_tag
|
30
|
+
assert_equal "019' Sexii_Shower 藤澤まお_洗白白", vs.title
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# -*- mode:ruby; coding:utf-8 -*-
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../../test_helper'
|
4
|
+
|
5
|
+
class TestYourFileHost < Test::Unit::TestCase
|
6
|
+
def test_scrape
|
7
|
+
vs = WWW::VideoScraper::YourFileHost.scrape('http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv', default_opt)
|
8
|
+
assert_equal 'http://www.yourfilehost.com/media.php?cat=video&file=XV436__03.wmv', vs.page_url
|
9
|
+
assert_match %r|http://cdn\w*\.yourfilehost\.com/unit1/flash8/\d+/[[:alnum:]]{32}\.flv|, vs.video_url
|
10
|
+
assert_match %r|http://cdn\w*\.yourfilehost\.com/thumbs/\d+/[[:alnum:]]{32}\.jpg|, vs.thumb_url
|
11
|
+
assert_equal 'XV436__03.wmv', vs.title
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
metadata
ADDED
@@ -0,0 +1,133 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: video_scraper
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.5
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- YAMAGUCHI Seiji
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-10-23 00:00:00 +09:00
|
13
|
+
default_executable:
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: mechanize
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 0.8.4
|
24
|
+
version:
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: hpricot
|
27
|
+
type: :runtime
|
28
|
+
version_requirement:
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.6.164
|
34
|
+
version:
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: json
|
37
|
+
type: :runtime
|
38
|
+
version_requirement:
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 1.1.3
|
44
|
+
version:
|
45
|
+
description: Web scraping library for video sharing sites.
|
46
|
+
email: valda@underscore.jp
|
47
|
+
executables: []
|
48
|
+
|
49
|
+
extensions: []
|
50
|
+
|
51
|
+
extra_rdoc_files:
|
52
|
+
- README
|
53
|
+
- ChangeLog
|
54
|
+
files:
|
55
|
+
- README
|
56
|
+
- ChangeLog
|
57
|
+
- Rakefile
|
58
|
+
- test/www/video_scraper/test_adult_satellites.rb
|
59
|
+
- test/www/video_scraper/test_base.rb
|
60
|
+
- test/www/video_scraper/test_pornotube.rb
|
61
|
+
- test/www/video_scraper/test_moro_tube.rb
|
62
|
+
- test/www/video_scraper/test_tube8.rb
|
63
|
+
- test/www/video_scraper/test_pornhub.rb
|
64
|
+
- test/www/video_scraper/test_your_file_host.rb
|
65
|
+
- test/www/video_scraper/test_nico_video.rb
|
66
|
+
- test/www/video_scraper/test_eic_book.rb
|
67
|
+
- test/www/video_scraper/test_you_porn.rb
|
68
|
+
- test/www/video_scraper/test_dailymotion.rb
|
69
|
+
- test/www/video_scraper/test_ameba_vision.rb
|
70
|
+
- test/www/video_scraper/test_age_sage.rb
|
71
|
+
- test/www/video_scraper/test_veoh.rb
|
72
|
+
- test/www/video_scraper/test_you_tube.rb
|
73
|
+
- test/www/video_scraper/test_red_tube.rb
|
74
|
+
- test/www/test_video_scraper.rb
|
75
|
+
- test/test_helper.rb
|
76
|
+
- lib/www/video_scraper/you_tube.rb
|
77
|
+
- lib/www/video_scraper/your_file_host.rb
|
78
|
+
- lib/www/video_scraper/red_tube.rb
|
79
|
+
- lib/www/video_scraper/dailymotion.rb
|
80
|
+
- lib/www/video_scraper/base.rb
|
81
|
+
- lib/www/video_scraper/you_porn.rb
|
82
|
+
- lib/www/video_scraper/moro_tube.rb
|
83
|
+
- lib/www/video_scraper/veoh.rb
|
84
|
+
- lib/www/video_scraper/eic_book.rb
|
85
|
+
- lib/www/video_scraper/nico_video.rb
|
86
|
+
- lib/www/video_scraper/ameba_vision.rb
|
87
|
+
- lib/www/video_scraper/adult_satellites.rb
|
88
|
+
- lib/www/video_scraper/pornotube.rb
|
89
|
+
- lib/www/video_scraper/tube8.rb
|
90
|
+
- lib/www/video_scraper/pornhub.rb
|
91
|
+
- lib/www/video_scraper/age_sage.rb
|
92
|
+
- lib/www/video_scraper.rb
|
93
|
+
has_rdoc: true
|
94
|
+
homepage: http://github.com/valda/video_scraper
|
95
|
+
licenses: []
|
96
|
+
|
97
|
+
post_install_message:
|
98
|
+
rdoc_options:
|
99
|
+
- --title
|
100
|
+
- video_scraper documentation
|
101
|
+
- --charset
|
102
|
+
- utf-8
|
103
|
+
- --opname
|
104
|
+
- index.html
|
105
|
+
- --line-numbers
|
106
|
+
- --main
|
107
|
+
- README
|
108
|
+
- --inline-source
|
109
|
+
- --exclude
|
110
|
+
- ^(examples|extras)/
|
111
|
+
require_paths:
|
112
|
+
- lib
|
113
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: "0"
|
118
|
+
version:
|
119
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - ">="
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: "0"
|
124
|
+
version:
|
125
|
+
requirements: []
|
126
|
+
|
127
|
+
rubyforge_project: video_scraper
|
128
|
+
rubygems_version: 1.3.5
|
129
|
+
signing_key:
|
130
|
+
specification_version: 3
|
131
|
+
summary: Web scraping library for video sharing sites.
|
132
|
+
test_files: []
|
133
|
+
|