hlspider 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
data/.gitignore ADDED
@@ -0,0 +1,4 @@
1
+ *.gem
2
+ .bundle
3
+ Gemfile.lock
4
+ pkg/*
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ gem 'eventmachine', :git => 'git://github.com/eventmachine/eventmachine.git', :require => 'eventmachine'
2
+ gem 'robinhood'
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
data/bin/hlspider ADDED
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ begin
4
+ require 'hlspider'
5
+ rescue LoadError
6
+ require 'rubygems'
7
+ require 'hlspider'
8
+ end
9
+
10
+ HLSpider::HLSpider.start
data/hlspider.gemspec ADDED
@@ -0,0 +1,21 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "hlspider/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "hlspider"
7
+ s.version = Hlspider::VERSION
8
+ s.platform = Gem::Platform::RUBY
9
+ s.authors = ["Brooke McKim"]
10
+ s.email = ["bmckim@telvue.com"]
11
+ s.homepage = ""
12
+ s.summary = %q{ASYNC .m3u8 downloader}
13
+ s.description = %q{Downloads .m3u8 playlist files and confirms their segments are properly aligned.}
14
+
15
+ s.rubyforge_project = "hlspider"
16
+
17
+ s.files = `git ls-files`.split("\n")
18
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
19
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
20
+ s.require_paths = ["lib"]
21
+ end
@@ -0,0 +1,27 @@
1
+ module HLSpider
2
+ module Line
3
+ def strip_query_params(line)
4
+ line.sub(/[?].{1,}/, '')
5
+ end
6
+
7
+ def get_filename(line)
8
+ line.slice(/\w{1,}(.ts)/)
9
+ end
10
+
11
+ def strip_all_but_file(line)
12
+ line.slice(/\w{1,}(.ts)/)
13
+ end
14
+
15
+ def strip_file(line)
16
+ line.sub(/\/\w{1,}(.)\w{1,}$/, "")
17
+ end
18
+
19
+ def relative_path?(line)
20
+ true if !line.match('http://')
21
+ end
22
+
23
+ def absolute_path?(line)
24
+ true if line.match('http://')
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,80 @@
1
+ module HLSpider
2
+ class Playlist
3
+ attr_accessor :file
4
+ attr_reader :playlists, :segments
5
+
6
+ def self.first_segment_numbers(files = [])
7
+ playlists = []
8
+ files.each do |f|
9
+ playlist = Playlist.new(f[:body])
10
+ playlists << {:title => f[:title], :playlist => playlist} if playlist.valid?
11
+ end
12
+
13
+ segments = []
14
+ playlists.each do |p|
15
+ segment = /(\d*.ts)/.match(p[:playlist].segments.first)[0].gsub(".ts", "")
16
+ segments << segment
17
+ end
18
+
19
+ return segments
20
+ end
21
+
22
+ def initialize(file)
23
+ @file = file
24
+
25
+ @valid = false
26
+ @variable_playlist = false
27
+ @segment_playlist = false
28
+
29
+ @playlists = []
30
+ @segments = []
31
+
32
+ parse
33
+ end
34
+
35
+ def variable_playlist?
36
+ @variable_playlist
37
+ end
38
+
39
+ def segment_playlist?
40
+ @segment_playlist
41
+ end
42
+
43
+ def valid?
44
+ @valid
45
+ end
46
+
47
+ def to_s
48
+ @file
49
+ end
50
+
51
+ private
52
+ include ::Line
53
+
54
+ def parse
55
+ @valid = true if /#EXTM3U/.match(@file)
56
+
57
+ if has_playlist?(@file) && !has_segment?(@file)
58
+ @variable_playlist = true
59
+ @file.each_line do |line|
60
+ @playlists << line.strip if has_playlist?(line)
61
+ end
62
+ elsif has_segment?(@file) && !has_playlist?(@file)
63
+ @segment_playlist = true
64
+ @file.each_line do |line|
65
+ @segments << line.strip if has_segment?(line)
66
+ end
67
+ else
68
+ @valid = false
69
+ end
70
+ end
71
+
72
+ def has_segment?(str)
73
+ true if /.ts/.match(str)
74
+ end
75
+
76
+ def has_playlist?(str)
77
+ true if /.m3u8/.match(str)
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,45 @@
1
+ require 'em-synchrony'
2
+ require 'em-synchrony/em-http'
3
+ require 'logger'
4
+
5
+ module HLSpider
6
+ class Spider
7
+ @@log = Logger.new('/tmp/hlspider.log')
8
+
9
+ def initialize(playlists)
10
+ @playlists = playlists
11
+ end
12
+
13
+ def crawl
14
+ EM.synchrony do
15
+ multi = EventMachine::Synchrony::Multi.new
16
+ @playlists.each_with_index do |p,i|
17
+ multi.add :"#{i}", EventMachine::HttpRequest.new(p).aget
18
+ @time = Time.now
19
+ end
20
+
21
+ res = multi.perform
22
+
23
+ responses = []
24
+ res.requests.each do |req|
25
+ responses << {:title => req.req.uri.to_s, :body => req.response}
26
+ end
27
+ segments = Playlist.first_segment_numbers(responses).uniq
28
+
29
+ if segments.size > 1 && ((segments[0].to_i - segments[1].to_i).abs > 1)
30
+ p "Segments are off #{@time}"
31
+ @@log.error "**********"
32
+ @@log.error segments.inspect
33
+ @@log.error "**********"
34
+ else
35
+ p "All Good. at #{segments[0]}"
36
+ @@log.debug "^^^^^^^^^^^"
37
+ @@log.debug segments.inspect
38
+ @@log.debug "^^^^^^^^^^^"
39
+ end
40
+
41
+ EventMachine.stop
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,3 @@
1
+ module Hlspider
2
+ VERSION = "0.0.1"
3
+ end
data/lib/hlspider.rb ADDED
@@ -0,0 +1,25 @@
1
+ path = File.expand_path(File.dirname(__FILE__))
2
+ ['line', 'spider', 'playlist'].each do |file|
3
+ require File.join(path, 'hlspider', file)
4
+ end
5
+
6
+ playlists = ["http://hls.telvue.com/hls/lomalinda/1-1/playlist.m3u8", "http://hls.telvue.com/hls/lomalinda/1-2/playlist.m3u8", "http://hls.telvue.com/hls/lomalinda/1-3/playlist.m3u8"]
7
+
8
+ while true
9
+ Spider.new(playlists).crawl
10
+ sleep 5
11
+ end
12
+ module HLSpider
13
+ class HLSpider < Thor
14
+
15
+ desc "crawl", "Crawl the specified playlists and make sure their segments align"
16
+ method_options :playlists, :type => :array, :default => [], :required => true
17
+ method_options :sleep, :type => :integer, :default => 5
18
+ def crawl
19
+ while true
20
+ Spider.new(playlists).crawl
21
+ sleep options[:sleep]
22
+ end
23
+ end
24
+ end
25
+ end
metadata ADDED
@@ -0,0 +1,66 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: hlspider
3
+ version: !ruby/object:Gem::Version
4
+ prerelease:
5
+ version: 0.0.1
6
+ platform: ruby
7
+ authors:
8
+ - Brooke McKim
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+
13
+ date: 2011-05-26 00:00:00 -04:00
14
+ default_executable:
15
+ dependencies: []
16
+
17
+ description: Downloads .m3u8 playlist files and confirms their segments are properly aligned.
18
+ email:
19
+ - bmckim@telvue.com
20
+ executables:
21
+ - hlspider
22
+ extensions: []
23
+
24
+ extra_rdoc_files: []
25
+
26
+ files:
27
+ - .gitignore
28
+ - Gemfile
29
+ - Rakefile
30
+ - bin/hlspider
31
+ - hlspider.gemspec
32
+ - lib/hlspider.rb
33
+ - lib/hlspider/line.rb
34
+ - lib/hlspider/playlist.rb
35
+ - lib/hlspider/spider.rb
36
+ - lib/hlspider/version.rb
37
+ has_rdoc: true
38
+ homepage: ""
39
+ licenses: []
40
+
41
+ post_install_message:
42
+ rdoc_options: []
43
+
44
+ require_paths:
45
+ - lib
46
+ required_ruby_version: !ruby/object:Gem::Requirement
47
+ none: false
48
+ requirements:
49
+ - - ">="
50
+ - !ruby/object:Gem::Version
51
+ version: "0"
52
+ required_rubygems_version: !ruby/object:Gem::Requirement
53
+ none: false
54
+ requirements:
55
+ - - ">="
56
+ - !ruby/object:Gem::Version
57
+ version: "0"
58
+ requirements: []
59
+
60
+ rubyforge_project: hlspider
61
+ rubygems_version: 1.6.2
62
+ signing_key:
63
+ specification_version: 3
64
+ summary: ASYNC .m3u8 downloader
65
+ test_files: []
66
+