hlspider 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,2 @@
+gem 'eventmachine', :git => 'git://github.com/eventmachine/eventmachine.git', :require => 'eventmachine'
+gem 'robinhood'

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/bin/hlspider

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'hlspider'
+rescue LoadError
+  require 'rubygems'
+  require 'hlspider'
+end
+
+HLSpider::HLSpider.start

data/hlspider.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "hlspider/version"
+
+Gem::Specification.new do |s|
+  s.name        = "hlspider"
+  s.version     = Hlspider::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Brooke McKim"]
+  s.email       = ["bmckim@telvue.com"]
+  s.homepage    = ""
+  s.summary     = %q{ASYNC .m3u8 downloader}
+  s.description = %q{Downloads .m3u8 playlist files and confirms their segments are properly aligned.}
+
+  s.rubyforge_project = "hlspider"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/hlspider/line.rb

@@ -0,0 +1,27 @@
+module HLSpider
+  module Line
+    def strip_query_params(line)
+      line.sub(/[?].{1,}/, '')
+    end
+      
+    def get_filename(line)
+     line.slice(/\w{1,}(.ts)/)
+    end
+
+    def strip_all_but_file(line)
+     line.slice(/\w{1,}(.ts)/)
+    end
+  
+    def strip_file(line)
+      line.sub(/\/\w{1,}(.)\w{1,}$/, "")
+    end  
+  
+    def relative_path?(line)
+     true if !line.match('http://')
+    end
+
+    def absolute_path?(line)
+     true if line.match('http://')
+    end
+  end
+end

data/lib/hlspider/playlist.rb

@@ -0,0 +1,80 @@
+module HLSpider
+  class Playlist
+    attr_accessor :file
+    attr_reader   :playlists, :segments
+  
+    def self.first_segment_numbers(files = [])
+      playlists = []
+      files.each do |f|
+        playlist = Playlist.new(f[:body])
+        playlists << {:title => f[:title], :playlist => playlist} if playlist.valid?
+      end  
+    
+      segments = []
+      playlists.each do |p|
+        segment = /(\d*.ts)/.match(p[:playlist].segments.first)[0].gsub(".ts", "")
+        segments << segment
+      end
+    
+      return segments         
+    end  
+  
+    def initialize(file)
+      @file = file
+  
+      @valid = false
+      @variable_playlist = false
+      @segment_playlist  = false
+    
+      @playlists = []
+      @segments  = []
+    
+      parse
+    end
+
+    def variable_playlist?
+      @variable_playlist
+    end  
+
+    def segment_playlist?
+      @segment_playlist
+    end  
+
+    def valid?
+      @valid
+    end  
+
+    def to_s
+      @file
+    end  
+  
+  private
+    include ::Line
+  
+    def parse
+      @valid = true if /#EXTM3U/.match(@file)
+    
+      if has_playlist?(@file) && !has_segment?(@file)
+        @variable_playlist = true 
+        @file.each_line do |line|
+          @playlists << line.strip if has_playlist?(line)
+        end  
+      elsif has_segment?(@file) && !has_playlist?(@file)
+        @segment_playlist  = true
+        @file.each_line do |line|
+          @segments << line.strip if has_segment?(line)
+        end 
+      else
+        @valid = false   
+      end
+    end
+  
+    def has_segment?(str)
+      true if /.ts/.match(str)
+    end
+  
+    def has_playlist?(str)
+      true if /.m3u8/.match(str)
+    end               
+  end
+end

data/lib/hlspider/spider.rb

@@ -0,0 +1,45 @@
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+require 'logger'
+
+module HLSpider
+  class Spider
+    @@log = Logger.new('/tmp/hlspider.log')
+  
+    def initialize(playlists)
+      @playlists = playlists
+    end  
+  
+    def crawl
+      EM.synchrony do
+        multi = EventMachine::Synchrony::Multi.new
+        @playlists.each_with_index do |p,i|
+          multi.add :"#{i}", EventMachine::HttpRequest.new(p).aget
+          @time = Time.now
+        end  
+    
+        res = multi.perform
+      
+        responses = []
+        res.requests.each do |req|
+          responses << {:title => req.req.uri.to_s, :body => req.response}
+        end
+        segments = Playlist.first_segment_numbers(responses).uniq
+      
+        if segments.size > 1 && ((segments[0].to_i - segments[1].to_i).abs > 1)
+          p "Segments are off #{@time}"
+          @@log.error "**********"
+          @@log.error segments.inspect
+          @@log.error "**********"        
+        else
+          p "All Good. at #{segments[0]}"
+          @@log.debug "^^^^^^^^^^^"
+          @@log.debug segments.inspect
+          @@log.debug "^^^^^^^^^^^"
+        end  
+
+        EventMachine.stop
+      end
+    end
+  end  
+end

data/lib/hlspider/version.rb

@@ -0,0 +1,3 @@
+module Hlspider
+  VERSION = "0.0.1"
+end

data/lib/hlspider.rb

@@ -0,0 +1,25 @@
+path = File.expand_path(File.dirname(__FILE__))
+['line', 'spider', 'playlist'].each do |file|
+  require File.join(path, 'hlspider', file)
+end
+
+playlists = ["http://hls.telvue.com/hls/lomalinda/1-1/playlist.m3u8", "http://hls.telvue.com/hls/lomalinda/1-2/playlist.m3u8", "http://hls.telvue.com/hls/lomalinda/1-3/playlist.m3u8"]
+
+while true
+  Spider.new(playlists).crawl
+  sleep 5
+end
+module HLSpider
+  class HLSpider < Thor
+    
+    desc "crawl", "Crawl the specified playlists and make sure their segments align"
+    method_options :playlists, :type => :array, :default => [], :required => true
+    method_options :sleep,     :type => :integer, :default => 5
+    def crawl
+      while true
+        Spider.new(playlists).crawl
+        sleep options[:sleep]
+      end  
+    end  
+  end  
+end  

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification 
+name: hlspider
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Brooke McKim
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-26 00:00:00 -04:00
+default_executable: 
+dependencies: []
+
+description: Downloads .m3u8 playlist files and confirms their segments are properly aligned.
+email: 
+- bmckim@telvue.com
+executables: 
+- hlspider
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- Rakefile
+- bin/hlspider
+- hlspider.gemspec
+- lib/hlspider.rb
+- lib/hlspider/line.rb
+- lib/hlspider/playlist.rb
+- lib/hlspider/spider.rb
+- lib/hlspider/version.rb
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: hlspider
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: ASYNC .m3u8 downloader
+test_files: []
+