hlspider 0.3.3 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.travis.yml +8 -0
- data/Gemfile +2 -0
- data/README.md +4 -13
- data/Rakefile +4 -2
- data/hlspider.gemspec +2 -6
- data/lib/hlspider/downloader.rb +29 -35
- data/lib/hlspider/response.rb +13 -0
- data/lib/hlspider/spider.rb +10 -14
- data/lib/hlspider/version.rb +1 -1
- data/spec/spec_helper.rb +1 -2
- metadata +11 -43
data/.travis.yml
ADDED
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
|
+
[![BuildStatus](https://travis-ci.org/brookemckim/hlspider.png)](https://travis-ci.org/brookemckim/hlspider)
|
2
|
+
[![CodeClimate](https://codeclimate.com/badge.png)](https://codeclimate.com/github/brookemckim/hlspider)
|
3
|
+
|
1
4
|
# HLSpider - the HTTP Live Streaming Spider
|
2
|
-
|
5
|
+
Downloads .m3u8 playlists and reports back on whether or not the playlists are aligned in time.
|
3
6
|
|
4
7
|
## Purpose
|
5
8
|
|
@@ -25,18 +28,6 @@ parent_url = "http://host.com/video1/all_bitrates_playlist.m3u8"
|
|
25
28
|
spider = HLSpider.new(parent_url)
|
26
29
|
```
|
27
30
|
|
28
|
-
```
|
29
|
-
spider.aligned?
|
30
|
-
spider.invalid_playlists
|
31
|
-
|
32
|
-
playlist = spider.playlists[0]
|
33
|
-
playlist.valid?
|
34
|
-
playlist.segments
|
35
|
-
playlist.url
|
36
|
-
playlist.file
|
37
|
-
playlist.target_duration
|
38
|
-
```
|
39
|
-
|
40
31
|
### Command line
|
41
32
|
|
42
33
|
```
|
data/Rakefile
CHANGED
@@ -1,5 +1,4 @@
|
|
1
|
-
require
|
2
|
-
|
1
|
+
require 'bundler/gem_tasks'
|
3
2
|
require 'rake/testtask'
|
4
3
|
|
5
4
|
Rake::TestTask.new do |t|
|
@@ -8,3 +7,6 @@ Rake::TestTask.new do |t|
|
|
8
7
|
t.test_files = FileList['spec/**/*_spec.rb']
|
9
8
|
t.verbose = true
|
10
9
|
end
|
10
|
+
|
11
|
+
task :default => [:test]
|
12
|
+
task :spec => [:test]
|
data/hlspider.gemspec
CHANGED
@@ -8,9 +8,8 @@ Gem::Specification.new do |s|
|
|
8
8
|
s.authors = ["brookemckim"]
|
9
9
|
s.email = ["brooke.mckim@gmail.com"]
|
10
10
|
s.homepage = "http://www.github.com/brookemckim/hlspider"
|
11
|
-
s.summary = %q{
|
12
|
-
s.description = %q{
|
13
|
-
|
11
|
+
s.summary = %q{Download and parse .m3u8 playlists.}
|
12
|
+
s.description = %q{Downloads .m3u8 playlists and reports back on whether or not the playlists are aligned in time.}
|
14
13
|
|
15
14
|
s.rubyforge_project = "hlspider"
|
16
15
|
|
@@ -20,8 +19,5 @@ Gem::Specification.new do |s|
|
|
20
19
|
s.require_paths = ["lib"]
|
21
20
|
|
22
21
|
# specify any dependencies here
|
23
|
-
s.add_runtime_dependency 'eventmachine', '~> 1.0.0.beta.4'
|
24
|
-
s.add_runtime_dependency 'em-http-request', '~> 1.0.0'
|
25
|
-
|
26
22
|
s.add_development_dependency 'minitest', '~> 2.7.0'
|
27
23
|
end
|
data/lib/hlspider/downloader.rb
CHANGED
@@ -1,50 +1,44 @@
|
|
1
|
-
|
2
|
-
require '
|
3
|
-
require 'em-http-request'
|
1
|
+
require 'net/http'
|
2
|
+
require 'hlspider/response'
|
4
3
|
|
4
|
+
# Internal: Asynchronsoly downloads urls and returns Array of responses.
|
5
5
|
module HLSpider
|
6
6
|
module Downloader
|
7
|
-
class
|
8
|
-
|
9
|
-
# Internal:
|
7
|
+
class DownloadError < StandardError; end;
|
8
|
+
|
9
|
+
# Internal: Download given URLs.
|
10
10
|
#
|
11
|
-
# urls
|
11
|
+
# urls - An Array of strings or a single string of URL(s)
|
12
12
|
#
|
13
13
|
# Examples
|
14
14
|
#
|
15
|
-
#
|
16
|
-
# # =>
|
15
|
+
# download(["http://www.google.com", "http://www.yahoo.com"])
|
16
|
+
# # => []
|
17
17
|
#
|
18
|
-
#
|
19
|
-
# # =>
|
18
|
+
# download("http://www.bing.com")
|
19
|
+
# # => []
|
20
20
|
#
|
21
21
|
# Returns the Array of responses.
|
22
|
-
# Raises HLSpider::Downloader::
|
23
|
-
# downloading all urls.
|
22
|
+
# Raises HLSpider::Downloader::DownloadError if there was a problem
|
23
|
+
# downloading all urls.
|
24
24
|
def download(urls)
|
25
25
|
urls = Array(urls)
|
26
|
-
|
27
|
-
responses = nil
|
28
|
-
EventMachine.run {
|
29
|
-
multi = EventMachine::MultiRequest.new
|
30
26
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
27
|
+
responses = []
|
28
|
+
threads = []
|
29
|
+
|
30
|
+
urls.each do |url|
|
31
|
+
threads << Thread.new {
|
32
|
+
uri = URI.parse(url)
|
33
|
+
body = Net::HTTP.get_response(uri).body
|
34
|
+
|
35
|
+
responses << Response.new(url, body)
|
36
|
+
}
|
36
37
|
|
37
|
-
|
38
|
-
|
39
|
-
EventMachine.stop
|
40
|
-
end
|
41
|
-
}
|
38
|
+
threads.each { |t| t.join }
|
39
|
+
end
|
42
40
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
41
|
+
responses
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
data/lib/hlspider/spider.rb
CHANGED
@@ -8,11 +8,6 @@
|
|
8
8
|
#
|
9
9
|
# Spider.new("http://host.tld/video1/parent_playlist.m3u8")
|
10
10
|
# # => #<HLSpider::Spider:0x10cab12d0>
|
11
|
-
|
12
|
-
require 'rubygems'
|
13
|
-
require 'eventmachine'
|
14
|
-
require 'em-http-request'
|
15
|
-
|
16
11
|
module HLSpider
|
17
12
|
class Spider
|
18
13
|
class InvalidPlaylist < StandardError; end;
|
@@ -100,22 +95,23 @@ module HLSpider
|
|
100
95
|
def dive(urls = [])
|
101
96
|
playlists = []
|
102
97
|
|
103
|
-
responses = download(urls)
|
104
|
-
|
105
|
-
|
98
|
+
responses = download(urls)
|
99
|
+
|
100
|
+
responses.each do |response|
|
101
|
+
playlist = Playlist.new(response.body, response.url)
|
106
102
|
|
107
|
-
if
|
108
|
-
if
|
109
|
-
playlists << dive(
|
103
|
+
if playlist.valid?
|
104
|
+
if playlist.variable_playlist?
|
105
|
+
playlists << dive(playlist.playlists)
|
110
106
|
else
|
111
|
-
playlists <<
|
107
|
+
playlists << playlist
|
112
108
|
end
|
113
109
|
else
|
114
|
-
raise InvalidPlaylist, "#{
|
110
|
+
raise InvalidPlaylist, "#{playlist.source} was an invalid playlist."
|
115
111
|
end
|
116
112
|
end
|
117
113
|
|
118
114
|
playlists.flatten
|
119
115
|
end
|
120
116
|
end
|
121
|
-
end
|
117
|
+
end
|
data/lib/hlspider/version.rb
CHANGED
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hlspider
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 15
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
-
|
10
|
-
version: 0.
|
8
|
+
- 4
|
9
|
+
- 0
|
10
|
+
version: 0.4.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- brookemckim
|
@@ -15,46 +15,12 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-
|
18
|
+
date: 2012-11-13 00:00:00 Z
|
19
19
|
dependencies:
|
20
|
-
- !ruby/object:Gem::Dependency
|
21
|
-
name: eventmachine
|
22
|
-
prerelease: false
|
23
|
-
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
-
none: false
|
25
|
-
requirements:
|
26
|
-
- - ~>
|
27
|
-
- !ruby/object:Gem::Version
|
28
|
-
hash: 62196363
|
29
|
-
segments:
|
30
|
-
- 1
|
31
|
-
- 0
|
32
|
-
- 0
|
33
|
-
- beta
|
34
|
-
- 4
|
35
|
-
version: 1.0.0.beta.4
|
36
|
-
type: :runtime
|
37
|
-
version_requirements: *id001
|
38
|
-
- !ruby/object:Gem::Dependency
|
39
|
-
name: em-http-request
|
40
|
-
prerelease: false
|
41
|
-
requirement: &id002 !ruby/object:Gem::Requirement
|
42
|
-
none: false
|
43
|
-
requirements:
|
44
|
-
- - ~>
|
45
|
-
- !ruby/object:Gem::Version
|
46
|
-
hash: 23
|
47
|
-
segments:
|
48
|
-
- 1
|
49
|
-
- 0
|
50
|
-
- 0
|
51
|
-
version: 1.0.0
|
52
|
-
type: :runtime
|
53
|
-
version_requirements: *id002
|
54
20
|
- !ruby/object:Gem::Dependency
|
55
21
|
name: minitest
|
56
22
|
prerelease: false
|
57
|
-
requirement: &
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
58
24
|
none: false
|
59
25
|
requirements:
|
60
26
|
- - ~>
|
@@ -66,8 +32,8 @@ dependencies:
|
|
66
32
|
- 0
|
67
33
|
version: 2.7.0
|
68
34
|
type: :development
|
69
|
-
version_requirements: *
|
70
|
-
description:
|
35
|
+
version_requirements: *id001
|
36
|
+
description: Downloads .m3u8 playlists and reports back on whether or not the playlists are aligned in time.
|
71
37
|
email:
|
72
38
|
- brooke.mckim@gmail.com
|
73
39
|
executables:
|
@@ -79,6 +45,7 @@ extra_rdoc_files: []
|
|
79
45
|
files:
|
80
46
|
- .gitignore
|
81
47
|
- .rbenv-version
|
48
|
+
- .travis.yml
|
82
49
|
- Gemfile
|
83
50
|
- LICENSE
|
84
51
|
- README.md
|
@@ -89,6 +56,7 @@ files:
|
|
89
56
|
- lib/hlspider/downloader.rb
|
90
57
|
- lib/hlspider/playlist.rb
|
91
58
|
- lib/hlspider/playlist_line.rb
|
59
|
+
- lib/hlspider/response.rb
|
92
60
|
- lib/hlspider/spider.rb
|
93
61
|
- lib/hlspider/version.rb
|
94
62
|
- spec/hlspider/playlist_line_spec.rb
|
@@ -127,7 +95,7 @@ rubyforge_project: hlspider
|
|
127
95
|
rubygems_version: 1.8.24
|
128
96
|
signing_key:
|
129
97
|
specification_version: 3
|
130
|
-
summary:
|
98
|
+
summary: Download and parse .m3u8 playlists.
|
131
99
|
test_files:
|
132
100
|
- spec/hlspider/playlist_line_spec.rb
|
133
101
|
- spec/hlspider/spider_spec.rb
|