hlspider 0.1.0 → 0.1.2

data/README.md

@@ -3,7 +3,10 @@ HLSpider
 ASYNC .m3u8 downloader. Downloads .m3u8 playlist files and confirms their segments are properly aligned.
 
 Usage
------  
+-----
+    hlspider crawl --playlists=http://site.tld/playlist1.m3u8 http://site.tld/playlist2.m3u8 http://site.tld/playlist3.m3u8
+
+  
     hlspider
 
     Tasks:
@@ -17,7 +20,7 @@ Usage
     Options:
     --playlists=one two three  
     [--sleep=N]  # Default: 5
-    [--log=LOG]  # Not implemented yet
+    [--log=LOG]  # Disabled by default - Log provides some extra debug information
   
 
   

data/lib/hlspider/playlist.rb

@@ -1,27 +1,12 @@
 module HLSpider
   class Playlist
-    attr_accessor :file
+    attr_accessor :file, :source
     attr_reader   :playlists, :segments
   
-    def self.first_segment_numbers(files = [])
-      playlists = []
-      files.each do |f|
-        playlist = Playlist.new(f[:body])
-        playlists << {:title => f[:title], :playlist => playlist} if playlist.valid?
-      end  
-    
-      segments = []
-      playlists.each do |p|
-        segment = /(\d*.ts)/.match(p[:playlist].segments.first)[0].gsub(".ts", "")
-        segments << segment
-      end
-    
-      return segments         
-    end  
-  
-    def initialize(file)
-      @file = file
-  
+    def initialize(file, source = '')
+      @file   = file
+      @source = source
+       
       @valid = false
       @variable_playlist = false
       @segment_playlist  = false
@@ -49,11 +34,8 @@ module HLSpider
     end  
   
   private
-    include Line
-  
     def parse
       @valid = true if /#EXTM3U/.match(@file)
-    
       if has_playlist?(@file) && !has_segment?(@file)
         @variable_playlist = true 
         @file.each_line do |line|
@@ -70,7 +52,7 @@ module HLSpider
     end
   
     def has_segment?(str)
-      true if /.ts/.match(str)
+      true if /#EXT-X-MEDIA-SEQUENCE/.match(str)
     end
   
     def has_playlist?(str)

data/lib/hlspider/spider.rb

@@ -1,6 +1,8 @@
+require 'rubygems'
 require 'em-synchrony'
 require 'em-synchrony/em-http'
 require 'logger'
+require_relative 'playlist'  
 
 module HLSpider
   class Spider
@@ -10,43 +12,73 @@ module HLSpider
     end  
   
     def crawl
+      playlists = dive(@playlists)
+    
+      segments = []
+      playlists.each { |p| segments << p.segments.first }
+      
+      puts Time.now
+      put_and_log "--- #{segments.inspect} ---"
+      segments.uniq!
+    
+      if segments.size > 1 && ((segments[0].to_i - segments[1].to_i).abs > 1)
+        p "Segments are off - #{@time}"
+        log "**********"
+        log segments.inspect
+        log "**********"
+      elsif segments.size == 1
+        p "All Good. at #{segments[0]}"
+        log "^^^^^^^^^^^"
+        log segments.inspect
+        log "^^^^^^^^^^^"
+      else  
+        p "No segements found - #{@time}"
+        log "~~~~~~~~~~~"
+        log "No segments!"
+        log "~~~~~~~~~~~"
+      end  
+    end
+    
+    private
+    
+    def async_download(urls = [])
+      res = nil
+      
       EM.synchrony do
         multi = EventMachine::Synchrony::Multi.new
-        @playlists.each_with_index do |p,i|
+        
+        urls.each_with_index do |p,i|
           log p
-          puts "Downloading: #{p}"
           multi.add :"#{i}", EventMachine::HttpRequest.new(p).aget
           @time = Time.now
         end  
-    
+  
         res = multi.perform
+    
+        EventMachine.stop         
+      end  
       
-        responses = []
-        res.requests.each do |req|
-          responses << {:title => req.req.uri.to_s, :body => req.response}
-        end
-        segments = Playlist.first_segment_numbers(responses)
-        puts Time.now
-        put_and_log "--- #{segments.inspect} ---"
-        segments.uniq!
+      return res   
+    end  
+    
+    def dive(urls = [])
+      playlists = []
+            
+      res = async_download(urls)              
+      res.requests.each do |req|
+        p = Playlist.new(req.response, req.req.uri.to_s)
       
-        if segments.size > 1 && ((segments[0].to_i - segments[1].to_i).abs > 1)
-          p "Segments are off #{@time}"
-          log "**********"
-          log segments.inspect
-          log "**********"
-        else
-          p "All Good. at #{segments[0]}"
-          log "^^^^^^^^^^^"
-          log segments.inspect
-          log "^^^^^^^^^^^"
+        if p.valid?
+          if p.variable_playlist?
+            playlists = dive(p.playlists)
+          else
+            playlists << p
+          end 
         end  
-
-        EventMachine.stop
       end
-    end
-    
-    private
+      
+      return playlists  
+    end  
     
     def log(str)
       #eval "@log.#{type} \"#{str}\"" if @log

data/lib/hlspider/version.rb

@@ -1,3 +1,3 @@
 module Hlspider
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

data/lib/hlspider.rb

@@ -18,6 +18,6 @@ module HLSpider
 end  
 
 path = File.expand_path(File.dirname(__FILE__))
-['line', 'spider', 'playlist'].each do |file|
+['spider', 'playlist'].each do |file|
   require File.join(path, 'hlspider', file)
 end

metadata

@@ -1,72 +1,70 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: hlspider
-version: !ruby/object:Gem::Version 
+version: !ruby/object:Gem::Version
+  version: 0.1.2
   prerelease: 
-  version: 0.1.0
 platform: ruby
-authors: 
+authors:
 - Brooke McKim
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-05-27 00:00:00 -04:00
+date: 2011-08-18 00:00:00.000000000 -04:00
 default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+dependencies:
+- !ruby/object:Gem::Dependency
   name: thor
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &2152994560 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 0.14.0
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: em-synchrony
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *2152994560
+- !ruby/object:Gem::Dependency
+  name: em-synchrony
+  requirement: &2152993020 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 0.3.0.beta.1
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: eventmachine
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *2152993020
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: &2152983760 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.0.0.beta.3
   type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: em-http-request
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: *2152983760
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: &2152981200 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.0.0.beta.3
   type: :runtime
-  version_requirements: *id004
-description: Downloads .m3u8 playlist files and confirms their segments are properly aligned.
-email: 
+  prerelease: false
+  version_requirements: *2152981200
+description: Downloads .m3u8 playlist files and confirms their segments are properly
+  aligned.
+email:
 - bmckim@telvue.com
-executables: 
+executables:
 - hlspider
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
 - Gemfile
 - README.md
@@ -81,30 +79,26 @@ files:
 has_rdoc: true
 homepage: http://github.com/telvue/hlspider
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: hlspider
 rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: ASYNC .m3u8 downloader
 test_files: []
-