hlspider 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +4 -0
- data/Gemfile +2 -0
- data/Rakefile +2 -0
- data/bin/hlspider +10 -0
- data/hlspider.gemspec +21 -0
- data/lib/hlspider/line.rb +27 -0
- data/lib/hlspider/playlist.rb +80 -0
- data/lib/hlspider/spider.rb +45 -0
- data/lib/hlspider/version.rb +3 -0
- data/lib/hlspider.rb +25 -0
- metadata +66 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Rakefile
ADDED
data/bin/hlspider
ADDED
data/hlspider.gemspec
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "hlspider/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "hlspider"
|
7
|
+
s.version = Hlspider::VERSION
|
8
|
+
s.platform = Gem::Platform::RUBY
|
9
|
+
s.authors = ["Brooke McKim"]
|
10
|
+
s.email = ["bmckim@telvue.com"]
|
11
|
+
s.homepage = ""
|
12
|
+
s.summary = %q{ASYNC .m3u8 downloader}
|
13
|
+
s.description = %q{Downloads .m3u8 playlist files and confirms their segments are properly aligned.}
|
14
|
+
|
15
|
+
s.rubyforge_project = "hlspider"
|
16
|
+
|
17
|
+
s.files = `git ls-files`.split("\n")
|
18
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
19
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
20
|
+
s.require_paths = ["lib"]
|
21
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module HLSpider
|
2
|
+
module Line
|
3
|
+
def strip_query_params(line)
|
4
|
+
line.sub(/[?].{1,}/, '')
|
5
|
+
end
|
6
|
+
|
7
|
+
def get_filename(line)
|
8
|
+
line.slice(/\w{1,}(.ts)/)
|
9
|
+
end
|
10
|
+
|
11
|
+
def strip_all_but_file(line)
|
12
|
+
line.slice(/\w{1,}(.ts)/)
|
13
|
+
end
|
14
|
+
|
15
|
+
def strip_file(line)
|
16
|
+
line.sub(/\/\w{1,}(.)\w{1,}$/, "")
|
17
|
+
end
|
18
|
+
|
19
|
+
def relative_path?(line)
|
20
|
+
true if !line.match('http://')
|
21
|
+
end
|
22
|
+
|
23
|
+
def absolute_path?(line)
|
24
|
+
true if line.match('http://')
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
module HLSpider
|
2
|
+
class Playlist
|
3
|
+
attr_accessor :file
|
4
|
+
attr_reader :playlists, :segments
|
5
|
+
|
6
|
+
def self.first_segment_numbers(files = [])
|
7
|
+
playlists = []
|
8
|
+
files.each do |f|
|
9
|
+
playlist = Playlist.new(f[:body])
|
10
|
+
playlists << {:title => f[:title], :playlist => playlist} if playlist.valid?
|
11
|
+
end
|
12
|
+
|
13
|
+
segments = []
|
14
|
+
playlists.each do |p|
|
15
|
+
segment = /(\d*.ts)/.match(p[:playlist].segments.first)[0].gsub(".ts", "")
|
16
|
+
segments << segment
|
17
|
+
end
|
18
|
+
|
19
|
+
return segments
|
20
|
+
end
|
21
|
+
|
22
|
+
def initialize(file)
|
23
|
+
@file = file
|
24
|
+
|
25
|
+
@valid = false
|
26
|
+
@variable_playlist = false
|
27
|
+
@segment_playlist = false
|
28
|
+
|
29
|
+
@playlists = []
|
30
|
+
@segments = []
|
31
|
+
|
32
|
+
parse
|
33
|
+
end
|
34
|
+
|
35
|
+
def variable_playlist?
|
36
|
+
@variable_playlist
|
37
|
+
end
|
38
|
+
|
39
|
+
def segment_playlist?
|
40
|
+
@segment_playlist
|
41
|
+
end
|
42
|
+
|
43
|
+
def valid?
|
44
|
+
@valid
|
45
|
+
end
|
46
|
+
|
47
|
+
def to_s
|
48
|
+
@file
|
49
|
+
end
|
50
|
+
|
51
|
+
private
|
52
|
+
include ::Line
|
53
|
+
|
54
|
+
def parse
|
55
|
+
@valid = true if /#EXTM3U/.match(@file)
|
56
|
+
|
57
|
+
if has_playlist?(@file) && !has_segment?(@file)
|
58
|
+
@variable_playlist = true
|
59
|
+
@file.each_line do |line|
|
60
|
+
@playlists << line.strip if has_playlist?(line)
|
61
|
+
end
|
62
|
+
elsif has_segment?(@file) && !has_playlist?(@file)
|
63
|
+
@segment_playlist = true
|
64
|
+
@file.each_line do |line|
|
65
|
+
@segments << line.strip if has_segment?(line)
|
66
|
+
end
|
67
|
+
else
|
68
|
+
@valid = false
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def has_segment?(str)
|
73
|
+
true if /.ts/.match(str)
|
74
|
+
end
|
75
|
+
|
76
|
+
def has_playlist?(str)
|
77
|
+
true if /.m3u8/.match(str)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'em-synchrony'
|
2
|
+
require 'em-synchrony/em-http'
|
3
|
+
require 'logger'
|
4
|
+
|
5
|
+
module HLSpider
|
6
|
+
class Spider
|
7
|
+
@@log = Logger.new('/tmp/hlspider.log')
|
8
|
+
|
9
|
+
def initialize(playlists)
|
10
|
+
@playlists = playlists
|
11
|
+
end
|
12
|
+
|
13
|
+
def crawl
|
14
|
+
EM.synchrony do
|
15
|
+
multi = EventMachine::Synchrony::Multi.new
|
16
|
+
@playlists.each_with_index do |p,i|
|
17
|
+
multi.add :"#{i}", EventMachine::HttpRequest.new(p).aget
|
18
|
+
@time = Time.now
|
19
|
+
end
|
20
|
+
|
21
|
+
res = multi.perform
|
22
|
+
|
23
|
+
responses = []
|
24
|
+
res.requests.each do |req|
|
25
|
+
responses << {:title => req.req.uri.to_s, :body => req.response}
|
26
|
+
end
|
27
|
+
segments = Playlist.first_segment_numbers(responses).uniq
|
28
|
+
|
29
|
+
if segments.size > 1 && ((segments[0].to_i - segments[1].to_i).abs > 1)
|
30
|
+
p "Segments are off #{@time}"
|
31
|
+
@@log.error "**********"
|
32
|
+
@@log.error segments.inspect
|
33
|
+
@@log.error "**********"
|
34
|
+
else
|
35
|
+
p "All Good. at #{segments[0]}"
|
36
|
+
@@log.debug "^^^^^^^^^^^"
|
37
|
+
@@log.debug segments.inspect
|
38
|
+
@@log.debug "^^^^^^^^^^^"
|
39
|
+
end
|
40
|
+
|
41
|
+
EventMachine.stop
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
data/lib/hlspider.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
path = File.expand_path(File.dirname(__FILE__))
|
2
|
+
['line', 'spider', 'playlist'].each do |file|
|
3
|
+
require File.join(path, 'hlspider', file)
|
4
|
+
end
|
5
|
+
|
6
|
+
playlists = ["http://hls.telvue.com/hls/lomalinda/1-1/playlist.m3u8", "http://hls.telvue.com/hls/lomalinda/1-2/playlist.m3u8", "http://hls.telvue.com/hls/lomalinda/1-3/playlist.m3u8"]
|
7
|
+
|
8
|
+
while true
|
9
|
+
Spider.new(playlists).crawl
|
10
|
+
sleep 5
|
11
|
+
end
|
12
|
+
module HLSpider
|
13
|
+
class HLSpider < Thor
|
14
|
+
|
15
|
+
desc "crawl", "Crawl the specified playlists and make sure their segments align"
|
16
|
+
method_options :playlists, :type => :array, :default => [], :required => true
|
17
|
+
method_options :sleep, :type => :integer, :default => 5
|
18
|
+
def crawl
|
19
|
+
while true
|
20
|
+
Spider.new(playlists).crawl
|
21
|
+
sleep options[:sleep]
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
metadata
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: hlspider
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease:
|
5
|
+
version: 0.0.1
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Brooke McKim
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2011-05-26 00:00:00 -04:00
|
14
|
+
default_executable:
|
15
|
+
dependencies: []
|
16
|
+
|
17
|
+
description: Downloads .m3u8 playlist files and confirms their segments are properly aligned.
|
18
|
+
email:
|
19
|
+
- bmckim@telvue.com
|
20
|
+
executables:
|
21
|
+
- hlspider
|
22
|
+
extensions: []
|
23
|
+
|
24
|
+
extra_rdoc_files: []
|
25
|
+
|
26
|
+
files:
|
27
|
+
- .gitignore
|
28
|
+
- Gemfile
|
29
|
+
- Rakefile
|
30
|
+
- bin/hlspider
|
31
|
+
- hlspider.gemspec
|
32
|
+
- lib/hlspider.rb
|
33
|
+
- lib/hlspider/line.rb
|
34
|
+
- lib/hlspider/playlist.rb
|
35
|
+
- lib/hlspider/spider.rb
|
36
|
+
- lib/hlspider/version.rb
|
37
|
+
has_rdoc: true
|
38
|
+
homepage: ""
|
39
|
+
licenses: []
|
40
|
+
|
41
|
+
post_install_message:
|
42
|
+
rdoc_options: []
|
43
|
+
|
44
|
+
require_paths:
|
45
|
+
- lib
|
46
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
47
|
+
none: false
|
48
|
+
requirements:
|
49
|
+
- - ">="
|
50
|
+
- !ruby/object:Gem::Version
|
51
|
+
version: "0"
|
52
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
53
|
+
none: false
|
54
|
+
requirements:
|
55
|
+
- - ">="
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
version: "0"
|
58
|
+
requirements: []
|
59
|
+
|
60
|
+
rubyforge_project: hlspider
|
61
|
+
rubygems_version: 1.6.2
|
62
|
+
signing_key:
|
63
|
+
specification_version: 3
|
64
|
+
summary: ASYNC .m3u8 downloader
|
65
|
+
test_files: []
|
66
|
+
|