hlspider 0.1.0 → 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
data/README.md CHANGED
@@ -3,7 +3,10 @@ HLSpider
3
3
  ASYNC .m3u8 downloader. Downloads .m3u8 playlist files and confirms their segments are properly aligned.
4
4
 
5
5
  Usage
6
- -----
6
+ -----
7
+ hlspider crawl --playlists=http://site.tld/playlist1.m3u8 http://site.tld/playlist2.m3u8 http://site.tld/playlist3.m3u8
8
+
9
+
7
10
  hlspider
8
11
 
9
12
  Tasks:
@@ -17,7 +20,7 @@ Usage
17
20
  Options:
18
21
  --playlists=one two three
19
22
  [--sleep=N] # Default: 5
20
- [--log=LOG] # Not implemented yet
23
+ [--log=LOG] # Disabled by default - Log provides some extra debug information
21
24
 
22
25
 
23
26
 
@@ -1,27 +1,12 @@
1
1
  module HLSpider
2
2
  class Playlist
3
- attr_accessor :file
3
+ attr_accessor :file, :source
4
4
  attr_reader :playlists, :segments
5
5
 
6
- def self.first_segment_numbers(files = [])
7
- playlists = []
8
- files.each do |f|
9
- playlist = Playlist.new(f[:body])
10
- playlists << {:title => f[:title], :playlist => playlist} if playlist.valid?
11
- end
12
-
13
- segments = []
14
- playlists.each do |p|
15
- segment = /(\d*.ts)/.match(p[:playlist].segments.first)[0].gsub(".ts", "")
16
- segments << segment
17
- end
18
-
19
- return segments
20
- end
21
-
22
- def initialize(file)
23
- @file = file
24
-
6
+ def initialize(file, source = '')
7
+ @file = file
8
+ @source = source
9
+
25
10
  @valid = false
26
11
  @variable_playlist = false
27
12
  @segment_playlist = false
@@ -49,11 +34,8 @@ module HLSpider
49
34
  end
50
35
 
51
36
  private
52
- include Line
53
-
54
37
  def parse
55
38
  @valid = true if /#EXTM3U/.match(@file)
56
-
57
39
  if has_playlist?(@file) && !has_segment?(@file)
58
40
  @variable_playlist = true
59
41
  @file.each_line do |line|
@@ -70,7 +52,7 @@ module HLSpider
70
52
  end
71
53
 
72
54
  def has_segment?(str)
73
- true if /.ts/.match(str)
55
+ true if /#EXT-X-MEDIA-SEQUENCE/.match(str)
74
56
  end
75
57
 
76
58
  def has_playlist?(str)
@@ -1,6 +1,8 @@
1
+ require 'rubygems'
1
2
  require 'em-synchrony'
2
3
  require 'em-synchrony/em-http'
3
4
  require 'logger'
5
+ require_relative 'playlist'
4
6
 
5
7
  module HLSpider
6
8
  class Spider
@@ -10,43 +12,73 @@ module HLSpider
10
12
  end
11
13
 
12
14
  def crawl
15
+ playlists = dive(@playlists)
16
+
17
+ segments = []
18
+ playlists.each { |p| segments << p.segments.first }
19
+
20
+ puts Time.now
21
+ put_and_log "--- #{segments.inspect} ---"
22
+ segments.uniq!
23
+
24
+ if segments.size > 1 && ((segments[0].to_i - segments[1].to_i).abs > 1)
25
+ p "Segments are off - #{@time}"
26
+ log "**********"
27
+ log segments.inspect
28
+ log "**********"
29
+ elsif segments.size == 1
30
+ p "All Good. at #{segments[0]}"
31
+ log "^^^^^^^^^^^"
32
+ log segments.inspect
33
+ log "^^^^^^^^^^^"
34
+ else
35
+ p "No segements found - #{@time}"
36
+ log "~~~~~~~~~~~"
37
+ log "No segments!"
38
+ log "~~~~~~~~~~~"
39
+ end
40
+ end
41
+
42
+ private
43
+
44
+ def async_download(urls = [])
45
+ res = nil
46
+
13
47
  EM.synchrony do
14
48
  multi = EventMachine::Synchrony::Multi.new
15
- @playlists.each_with_index do |p,i|
49
+
50
+ urls.each_with_index do |p,i|
16
51
  log p
17
- puts "Downloading: #{p}"
18
52
  multi.add :"#{i}", EventMachine::HttpRequest.new(p).aget
19
53
  @time = Time.now
20
54
  end
21
-
55
+
22
56
  res = multi.perform
57
+
58
+ EventMachine.stop
59
+ end
23
60
 
24
- responses = []
25
- res.requests.each do |req|
26
- responses << {:title => req.req.uri.to_s, :body => req.response}
27
- end
28
- segments = Playlist.first_segment_numbers(responses)
29
- puts Time.now
30
- put_and_log "--- #{segments.inspect} ---"
31
- segments.uniq!
61
+ return res
62
+ end
63
+
64
+ def dive(urls = [])
65
+ playlists = []
66
+
67
+ res = async_download(urls)
68
+ res.requests.each do |req|
69
+ p = Playlist.new(req.response, req.req.uri.to_s)
32
70
 
33
- if segments.size > 1 && ((segments[0].to_i - segments[1].to_i).abs > 1)
34
- p "Segments are off #{@time}"
35
- log "**********"
36
- log segments.inspect
37
- log "**********"
38
- else
39
- p "All Good. at #{segments[0]}"
40
- log "^^^^^^^^^^^"
41
- log segments.inspect
42
- log "^^^^^^^^^^^"
71
+ if p.valid?
72
+ if p.variable_playlist?
73
+ playlists = dive(p.playlists)
74
+ else
75
+ playlists << p
76
+ end
43
77
  end
44
-
45
- EventMachine.stop
46
78
  end
47
- end
48
-
49
- private
79
+
80
+ return playlists
81
+ end
50
82
 
51
83
  def log(str)
52
84
  #eval "@log.#{type} \"#{str}\"" if @log
@@ -1,3 +1,3 @@
1
1
  module Hlspider
2
- VERSION = "0.1.0"
2
+ VERSION = "0.1.2"
3
3
  end
data/lib/hlspider.rb CHANGED
@@ -18,6 +18,6 @@ module HLSpider
18
18
  end
19
19
 
20
20
  path = File.expand_path(File.dirname(__FILE__))
21
- ['line', 'spider', 'playlist'].each do |file|
21
+ ['spider', 'playlist'].each do |file|
22
22
  require File.join(path, 'hlspider', file)
23
23
  end
metadata CHANGED
@@ -1,72 +1,70 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: hlspider
3
- version: !ruby/object:Gem::Version
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.2
4
5
  prerelease:
5
- version: 0.1.0
6
6
  platform: ruby
7
- authors:
7
+ authors:
8
8
  - Brooke McKim
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
-
13
- date: 2011-05-27 00:00:00 -04:00
12
+ date: 2011-08-18 00:00:00.000000000 -04:00
14
13
  default_executable:
15
- dependencies:
16
- - !ruby/object:Gem::Dependency
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
17
16
  name: thor
18
- prerelease: false
19
- requirement: &id001 !ruby/object:Gem::Requirement
17
+ requirement: &2152994560 !ruby/object:Gem::Requirement
20
18
  none: false
21
- requirements:
22
- - - ">="
23
- - !ruby/object:Gem::Version
19
+ requirements:
20
+ - - ! '>='
21
+ - !ruby/object:Gem::Version
24
22
  version: 0.14.0
25
23
  type: :runtime
26
- version_requirements: *id001
27
- - !ruby/object:Gem::Dependency
28
- name: em-synchrony
29
24
  prerelease: false
30
- requirement: &id002 !ruby/object:Gem::Requirement
25
+ version_requirements: *2152994560
26
+ - !ruby/object:Gem::Dependency
27
+ name: em-synchrony
28
+ requirement: &2152993020 !ruby/object:Gem::Requirement
31
29
  none: false
32
- requirements:
33
- - - ">="
34
- - !ruby/object:Gem::Version
30
+ requirements:
31
+ - - ! '>='
32
+ - !ruby/object:Gem::Version
35
33
  version: 0.3.0.beta.1
36
34
  type: :runtime
37
- version_requirements: *id002
38
- - !ruby/object:Gem::Dependency
39
- name: eventmachine
40
35
  prerelease: false
41
- requirement: &id003 !ruby/object:Gem::Requirement
36
+ version_requirements: *2152993020
37
+ - !ruby/object:Gem::Dependency
38
+ name: eventmachine
39
+ requirement: &2152983760 !ruby/object:Gem::Requirement
42
40
  none: false
43
- requirements:
44
- - - ">="
45
- - !ruby/object:Gem::Version
41
+ requirements:
42
+ - - ! '>='
43
+ - !ruby/object:Gem::Version
46
44
  version: 1.0.0.beta.3
47
45
  type: :runtime
48
- version_requirements: *id003
49
- - !ruby/object:Gem::Dependency
50
- name: em-http-request
51
46
  prerelease: false
52
- requirement: &id004 !ruby/object:Gem::Requirement
47
+ version_requirements: *2152983760
48
+ - !ruby/object:Gem::Dependency
49
+ name: em-http-request
50
+ requirement: &2152981200 !ruby/object:Gem::Requirement
53
51
  none: false
54
- requirements:
55
- - - ">="
56
- - !ruby/object:Gem::Version
52
+ requirements:
53
+ - - ! '>='
54
+ - !ruby/object:Gem::Version
57
55
  version: 1.0.0.beta.3
58
56
  type: :runtime
59
- version_requirements: *id004
60
- description: Downloads .m3u8 playlist files and confirms their segments are properly aligned.
61
- email:
57
+ prerelease: false
58
+ version_requirements: *2152981200
59
+ description: Downloads .m3u8 playlist files and confirms their segments are properly
60
+ aligned.
61
+ email:
62
62
  - bmckim@telvue.com
63
- executables:
63
+ executables:
64
64
  - hlspider
65
65
  extensions: []
66
-
67
66
  extra_rdoc_files: []
68
-
69
- files:
67
+ files:
70
68
  - .gitignore
71
69
  - Gemfile
72
70
  - README.md
@@ -81,30 +79,26 @@ files:
81
79
  has_rdoc: true
82
80
  homepage: http://github.com/telvue/hlspider
83
81
  licenses: []
84
-
85
82
  post_install_message:
86
83
  rdoc_options: []
87
-
88
- require_paths:
84
+ require_paths:
89
85
  - lib
90
- required_ruby_version: !ruby/object:Gem::Requirement
86
+ required_ruby_version: !ruby/object:Gem::Requirement
91
87
  none: false
92
- requirements:
93
- - - ">="
94
- - !ruby/object:Gem::Version
95
- version: "0"
96
- required_rubygems_version: !ruby/object:Gem::Requirement
88
+ requirements:
89
+ - - ! '>='
90
+ - !ruby/object:Gem::Version
91
+ version: '0'
92
+ required_rubygems_version: !ruby/object:Gem::Requirement
97
93
  none: false
98
- requirements:
99
- - - ">="
100
- - !ruby/object:Gem::Version
101
- version: "0"
94
+ requirements:
95
+ - - ! '>='
96
+ - !ruby/object:Gem::Version
97
+ version: '0'
102
98
  requirements: []
103
-
104
99
  rubyforge_project: hlspider
105
100
  rubygems_version: 1.6.2
106
101
  signing_key:
107
102
  specification_version: 3
108
103
  summary: ASYNC .m3u8 downloader
109
104
  test_files: []
110
-