hlspider 0.0.6 → 0.0.13
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +3 -1
- data/hlspider.gemspec +2 -3
- data/lib/hlspider/spider.rb +20 -10
- data/lib/hlspider/version.rb +1 -1
- data/lib/hlspider.rb +3 -2
- metadata +2 -2
data/Gemfile
CHANGED
data/hlspider.gemspec
CHANGED
@@ -19,9 +19,8 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
20
20
|
s.require_paths = ["lib"]
|
21
21
|
|
22
|
-
s.add_dependency('thor')
|
22
|
+
s.add_dependency('thor', '>= 0.14.0')
|
23
23
|
s.add_dependency('em-synchrony', '>= 0.3.0.beta.1')
|
24
24
|
s.add_dependency('eventmachine', '>= 1.0.0.beta.3')
|
25
|
-
s.add_dependency('em-http-request', '>= 1.0.0.beta.3')
|
26
|
-
|
25
|
+
s.add_dependency('em-http-request', '>= 1.0.0.beta.3')
|
27
26
|
end
|
data/lib/hlspider/spider.rb
CHANGED
@@ -4,16 +4,17 @@ require 'logger'
|
|
4
4
|
|
5
5
|
module HLSpider
|
6
6
|
class Spider
|
7
|
-
|
8
|
-
|
9
|
-
def initialize(playlists)
|
7
|
+
def initialize(playlists, logfile = "")
|
10
8
|
@playlists = playlists
|
9
|
+
@log = Logger.new(logfile) unless logfile.empty?
|
11
10
|
end
|
12
11
|
|
13
12
|
def crawl
|
14
13
|
EM.synchrony do
|
15
14
|
multi = EventMachine::Synchrony::Multi.new
|
16
15
|
@playlists.each_with_index do |p,i|
|
16
|
+
log p
|
17
|
+
puts "Downloading: #{p}"
|
17
18
|
multi.add :"#{i}", EventMachine::HttpRequest.new(p).aget
|
18
19
|
@time = Time.now
|
19
20
|
end
|
@@ -24,22 +25,31 @@ module HLSpider
|
|
24
25
|
res.requests.each do |req|
|
25
26
|
responses << {:title => req.req.uri.to_s, :body => req.response}
|
26
27
|
end
|
27
|
-
segments = Playlist.first_segment_numbers(responses)
|
28
|
+
segments = Playlist.first_segment_numbers(responses)
|
29
|
+
log "--- #{segments.inspect} ---"
|
30
|
+
segments.uniq!
|
28
31
|
|
29
32
|
if segments.size > 1 && ((segments[0].to_i - segments[1].to_i).abs > 1)
|
30
33
|
p "Segments are off #{@time}"
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
+
log "**********", "error"
|
35
|
+
log segments.inspect, "error"
|
36
|
+
log "**********", "error"
|
34
37
|
else
|
35
38
|
p "All Good. at #{segments[0]}"
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
+
log "^^^^^^^^^^^"
|
40
|
+
log segments.inspect
|
41
|
+
log "^^^^^^^^^^^"
|
39
42
|
end
|
40
43
|
|
41
44
|
EventMachine.stop
|
42
45
|
end
|
43
46
|
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def log(str, type = 'info')
|
51
|
+
#need to work on this eval
|
52
|
+
#eval "@log.#{type} #{str}" if @log
|
53
|
+
end
|
44
54
|
end
|
45
55
|
end
|
data/lib/hlspider/version.rb
CHANGED
data/lib/hlspider.rb
CHANGED
@@ -5,11 +5,12 @@ module HLSpider
|
|
5
5
|
class HLSpider < ::Thor
|
6
6
|
|
7
7
|
desc "crawl", "Crawl the specified playlists and make sure their segments align"
|
8
|
-
method_option :playlists, :type => :array,
|
8
|
+
method_option :playlists, :type => :array, :default => [], :required => true
|
9
9
|
method_option :sleep, :type => :numeric, :default => 5
|
10
|
+
method_option :log, :type => :string, :default => ""
|
10
11
|
def crawl
|
11
12
|
while true
|
12
|
-
Spider.new(playlists).crawl
|
13
|
+
Spider.new(options[:playlists], options[:log]).crawl
|
13
14
|
sleep options[:sleep]
|
14
15
|
end
|
15
16
|
end
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: hlspider
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.0.
|
5
|
+
version: 0.0.13
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Brooke McKim
|
@@ -21,7 +21,7 @@ dependencies:
|
|
21
21
|
requirements:
|
22
22
|
- - ">="
|
23
23
|
- !ruby/object:Gem::Version
|
24
|
-
version:
|
24
|
+
version: 0.14.0
|
25
25
|
type: :runtime
|
26
26
|
version_requirements: *id001
|
27
27
|
- !ruby/object:Gem::Dependency
|