hlspider 0.0.6 → 0.0.13

Sign up to get free protection for your applications and to get access to all the features.
data/Gemfile CHANGED
@@ -1,2 +1,4 @@
1
1
  gem 'eventmachine', :git => 'git://github.com/eventmachine/eventmachine.git', :require => 'eventmachine'
2
- gem 'robinhood'
2
+ gem 'thor', '>= 0.14.0'
3
+ gem 'em-synchrony', '>= 0.3.0.beta.1'
4
+ gem 'em-http-request', '>= 1.0.0.beta.3'
data/hlspider.gemspec CHANGED
@@ -19,9 +19,8 @@ Gem::Specification.new do |s|
19
19
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
20
20
  s.require_paths = ["lib"]
21
21
 
22
- s.add_dependency('thor')
22
+ s.add_dependency('thor', '>= 0.14.0')
23
23
  s.add_dependency('em-synchrony', '>= 0.3.0.beta.1')
24
24
  s.add_dependency('eventmachine', '>= 1.0.0.beta.3')
25
- s.add_dependency('em-http-request', '>= 1.0.0.beta.3')
26
-
25
+ s.add_dependency('em-http-request', '>= 1.0.0.beta.3')
27
26
  end
@@ -4,16 +4,17 @@ require 'logger'
4
4
 
5
5
  module HLSpider
6
6
  class Spider
7
- @@log = Logger.new('/tmp/hlspider.log')
8
-
9
- def initialize(playlists)
7
+ def initialize(playlists, logfile = "")
10
8
  @playlists = playlists
9
+ @log = Logger.new(logfile) unless logfile.empty?
11
10
  end
12
11
 
13
12
  def crawl
14
13
  EM.synchrony do
15
14
  multi = EventMachine::Synchrony::Multi.new
16
15
  @playlists.each_with_index do |p,i|
16
+ log p
17
+ puts "Downloading: #{p}"
17
18
  multi.add :"#{i}", EventMachine::HttpRequest.new(p).aget
18
19
  @time = Time.now
19
20
  end
@@ -24,22 +25,31 @@ module HLSpider
24
25
  res.requests.each do |req|
25
26
  responses << {:title => req.req.uri.to_s, :body => req.response}
26
27
  end
27
- segments = Playlist.first_segment_numbers(responses).uniq
28
+ segments = Playlist.first_segment_numbers(responses)
29
+ log "--- #{segments.inspect} ---"
30
+ segments.uniq!
28
31
 
29
32
  if segments.size > 1 && ((segments[0].to_i - segments[1].to_i).abs > 1)
30
33
  p "Segments are off #{@time}"
31
- @@log.error "**********"
32
- @@log.error segments.inspect
33
- @@log.error "**********"
34
+ log "**********", "error"
35
+ log segments.inspect, "error"
36
+ log "**********", "error"
34
37
  else
35
38
  p "All Good. at #{segments[0]}"
36
- @@log.debug "^^^^^^^^^^^"
37
- @@log.debug segments.inspect
38
- @@log.debug "^^^^^^^^^^^"
39
+ log "^^^^^^^^^^^"
40
+ log segments.inspect
41
+ log "^^^^^^^^^^^"
39
42
  end
40
43
 
41
44
  EventMachine.stop
42
45
  end
43
46
  end
47
+
48
+ private
49
+
50
+ def log(str, type = 'info')
51
+ #need to work on this eval
52
+ #eval "@log.#{type} #{str}" if @log
53
+ end
44
54
  end
45
55
  end
@@ -1,3 +1,3 @@
1
1
  module Hlspider
2
- VERSION = "0.0.6"
2
+ VERSION = "0.0.13"
3
3
  end
data/lib/hlspider.rb CHANGED
@@ -5,11 +5,12 @@ module HLSpider
5
5
  class HLSpider < ::Thor
6
6
 
7
7
  desc "crawl", "Crawl the specified playlists and make sure their segments align"
8
- method_option :playlists, :type => :array, :default => [], :required => true
8
+ method_option :playlists, :type => :array, :default => [], :required => true
9
9
  method_option :sleep, :type => :numeric, :default => 5
10
+ method_option :log, :type => :string, :default => ""
10
11
  def crawl
11
12
  while true
12
- Spider.new(playlists).crawl
13
+ Spider.new(options[:playlists], options[:log]).crawl
13
14
  sleep options[:sleep]
14
15
  end
15
16
  end
metadata CHANGED
@@ -2,7 +2,7 @@
2
2
  name: hlspider
3
3
  version: !ruby/object:Gem::Version
4
4
  prerelease:
5
- version: 0.0.6
5
+ version: 0.0.13
6
6
  platform: ruby
7
7
  authors:
8
8
  - Brooke McKim
@@ -21,7 +21,7 @@ dependencies:
21
21
  requirements:
22
22
  - - ">="
23
23
  - !ruby/object:Gem::Version
24
- version: "0"
24
+ version: 0.14.0
25
25
  type: :runtime
26
26
  version_requirements: *id001
27
27
  - !ruby/object:Gem::Dependency