iplayer-dl 0.1.16

data/COPYING

@@ -0,0 +1,26 @@
+Copyright (c) 2008 Paul Battley <pbattley@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+If you have received this program as a self-executing bundle, the following
+licences apply to individual components:
+
+ruby: GNU General Public License (GPL), version 2; Ruby License
+rubyscript2exe: GNU General Public License (GPL), version 2
+wxruby: wxWindows Licence

data/README

@@ -0,0 +1,14 @@
+I've refactored iplayer-dl in order to make it easier to extend and wrap. As a
+direct result, it's no longer just one script. As a result, you'll need to
+install the libraries and command-line script in order to use it.
+
+The good news is that this is as easy as typing:
+
+  ruby setup.rb config
+  sudo ruby setup.rb install
+
+Windows users should omit 'sudo' in the second line.
+
+You can then run the downloader just by typing:
+
+  iplayer-dl

data/Rakefile

@@ -0,0 +1,95 @@
+require 'rake/testtask'
+require 'rake/packagetask'
+require 'rake/rdoctask'
+require 'rake'
+require 'find'
+require 'lib/iplayer/version'
+
+# Globals
+
+PKG_NAME = 'iplayer-dl'
+
+PKG_FILES = %w[ COPYING README setup.rb Rakefile ]
+Find.find('lib/', 'test/', 'bin/', 'share/') do |f|
+  if FileTest.directory?(f) and File.basename(f) =~ /^\./
+    Find.prune
+  else
+    PKG_FILES << f
+  end
+end
+
+EXE_FILES = PKG_FILES + %w[ application.ico init.rb ]
+
+# Tasks
+
+task :default => :test
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/test_*.rb']
+end
+
+Rake::PackageTask.new(PKG_NAME, IPlayer::VERSION) do |p|
+  p.need_tar_gz = true
+  p.package_files = PKG_FILES
+end
+
+desc "Build a Windows executable. Needs rubyscript2exe.rb in the current path and the wx gem installed."
+task :exe do |t|
+  mkdir_p 'tmp'
+  build_dir = File.join('tmp', "ipdl-#{IPlayer::GUI_VERSION}")
+  rm_rf build_dir
+  mkdir build_dir
+  EXE_FILES.each do |file|
+    next if File.directory?(file)
+    loc = File.join(build_dir, File.dirname(file))
+    mkdir_p loc
+    cp_r file, loc
+  end
+  sh "ruby rubyscript2exe.rb #{build_dir} --rubyscript2exe-verbose --rubyscript2exe-rubyw"
+  rm_rf build_dir
+  mkdir_p 'pkg'
+  mv "ipdl-#{IPlayer::GUI_VERSION}.exe", "pkg"
+end
+
+begin
+  require "rake/gempackagetask"
+
+  spec = Gem::Specification.new do |s|
+    # Change these as appropriate
+    s.name              = "iplayer-dl"
+    s.version           = IPlayer::VERSION
+    s.summary           = "Download iPlayer content"
+    s.author            = "Paul Battley"
+    s.email             = "pbattley@gmail.com"
+    s.homepage          = "http://po-ru.com/projects/iplayer-downloader"
+
+    s.has_rdoc          = false
+
+    # Add any extra files to include in the gem
+    s.files             = PKG_FILES
+    s.executables       = ["iplayer-dl"]
+
+    s.require_paths     = ["lib"]
+
+    # If you want to depend on other gems, add them here, along with any
+    # relevant versions
+    # s.add_dependency("some_other_gem", "~> 0.1.0")
+
+    # If your tests use any gems, include them here
+    s.add_development_dependency("mocha")
+  end
+
+  # This task actually builds the gem. We also regenerate a static
+  # .gemspec file, which is useful if something (i.e. GitHub) will
+  # be automatically building a gem for this project. If you're not
+  # using GitHub, edit as appropriate.
+  Rake::GemPackageTask.new(spec) do |pkg|
+    pkg.gem_spec = spec
+
+    # Generate the gemspec file for github.
+    file = File.dirname(__FILE__) + "/#{spec.name}.gemspec"
+    File.open(file, "w") {|f| f << spec.to_ruby }
+  end
+rescue LoadError
+end

data/bin/iplayer-dl

@@ -0,0 +1,152 @@
+#!/usr/bin/env ruby
+
+# Download iPlayer programmes by spoofing an iPhone
+# Paul Battley - http://po-ru.com/
+#
+# Get the latest version via subversion:
+# svn co http://paulbattley.googlecode.com/svn/iplayer-dl
+
+require 'iplayer'
+require 'optparse'
+require 'fileutils'
+require 'iplayer/version'
+require 'iplayer/preferences'
+
+include IPlayer
+include IPlayer::Errors
+
+preferences = IPlayer::Preferences.new
+filenames   = []
+pid_list    = nil
+dry_run     = false
+
+opts = ARGV.options{ |o|
+  o.banner << ' IDENTIFIER [IDENTIFIER [...]]'
+  o.define_head 'Download DRM-free videos from the BBC iPlayer, courtesy of their iPhone interface.'
+  o.separator 'IDENTIFIER is the iPlayer viewing page URL or the PID of the programme.'
+  o.separator ''
+  o.on(
+    '-t', '--type-preference=VERSION', String,
+    'Video types in order of preference.',
+    "Default is '#{preferences.type_preference.join(',')}'."
+  ) { |s| preferences.type_preference = s.split(/,\s*/) }
+  o.on(
+    '-d', '--download-path=PATH', String,
+    'Location into which downloaded files will be saved.',
+    'Default is current working directory.'
+  ) { |d| preferences.download_path = d }
+  o.on(
+    '-s', '--title-subdir',
+    'Place downloaded files in a sub-directory named after the title of the programme.'
+  ) { preferences.subdirs = true }
+  o.on(
+    '-u', '--subtitles',
+    'Also download subtitles.'
+  ) { preferences.subtitles = true }
+  o.on(
+    '-f', '--filename=FILENAME', String,
+    'Manually specify a name for the downloaded file.',
+    'The default is constructed from the programme metadata.',
+    'You can specify this multiple times for each download in order.'
+  ) { |f| filenames << f }
+  o.on(
+    '-p', '--http-proxy=HOST:PORT', String,
+    'Specify an HTTP proxy.',
+    'Default is taken from the http_proxy environment variable.'
+  ) { |p| preferences.http_proxy = p }
+  o.on(
+    '-l', '--pid-list=FILENAME', String,
+    'List PIDs to be downloaded in a file, one per line.'
+  ) { |l| pid_list = l }
+  o.on(
+    '-n', '--dry-run', String,
+    'Parse the iPlayer page and output the filename but don\'t actually download (useful for scripting).'
+  ) { |n| dry_run = true }
+  o.on(
+    '-v', '--version',
+    'Show the software version.'
+  ) { puts IPlayer::VERSION; exit }
+  o.on_tail(
+    '-h', '--help', 
+    'Show this help message.'
+  ) { puts o; exit }
+}
+
+begin
+  opts.parse!
+  pids = ARGV
+  if pid_list
+    pids += File.read(pid_list).strip.split(/\s*\r?\n\s*/)
+  end
+  raise 'no programme identifier specified' if pids.empty?
+rescue => exception
+  $stderr.puts 'Error: '+exception, ''
+  puts opts
+  exit 1
+end
+
+if http_proxy = preferences.http_proxy
+  http_proxy = 'http://' + http_proxy unless http_proxy =~ %r{^http://}
+  u = URI.parse(http_proxy)
+  $stderr.puts "Using proxy #{u.host}:#{u.port}"
+  http = Net::HTTP::Proxy(u.host, u.port)
+else
+  http = Net::HTTP
+end
+
+pids.each_with_index do |pid, i|
+  browser = Browser.new(http)
+
+  begin
+    pid = Downloader.extract_pid(pid)
+    downloader = Downloader.new(browser, pid)
+
+    available_versions = downloader.available_versions
+    raise MP4Unavailable if available_versions.empty?
+    version = available_versions.sort_by{ |v| 
+      preferences.type_preference.index(v.name) || 100 
+    }.first
+
+    filename = filenames[i]
+    if filename.nil? || preferences.subdirs
+      metadata = downloader.metadata
+      filename ||= "#{ metadata.full_title }.#{ metadata.filetype }".gsub(/[^a-z0-9 \-\.]+/i, '')
+    end
+    if preferences.subdirs
+      subdir = metadata.title.gsub(/[^a-z0-9 \-\.]+/i, '')
+      path = File.expand_path( File.join( preferences.download_path, subdir ))
+      FileUtils.makedirs(path)
+      path = File.join( path, filename )
+    else
+      path = File.expand_path( File.join( preferences.download_path, filename ))
+    end
+
+    if dry_run
+      $stdout.puts filename
+    else
+      old_percentage = nil
+      first_chunk = true
+
+      $stderr.puts "#{ filename } (version: #{ version.name })"
+      download_options = {:subtitles => preferences.subtitles}
+      downloader.download(version.pid, path, download_options) do |position, max|
+        if first_chunk
+          $stderr.puts "Resuming download at #{position} bytes." if position > 0
+          first_chunk = false
+        end
+        
+        percentage = "%.1f" % [((1000 * position) / max) / 10.0]
+        if percentage != old_percentage
+          old_percentage = percentage
+          $stderr.print "\r#{ percentage }%"
+          $stderr.flush
+        end
+      end
+      $stderr.puts
+    end
+
+  rescue RecognizedError => error
+    $stderr.puts(error.to_str)
+    next
+  end
+end

data/bin/iplayer-dl-gui

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+
+# Download iPlayer programmes by spoofing an iPhone
+# WxWidgets GUI interface
+# Paul Battley - http://po-ru.com/
+#
+# Get the latest version via subversion:
+# svn co http://paulbattley.googlecode.com/svn/iplayer-dl
+
+require 'iplayer'
+require 'iplayer/gui/app'
+require 'iplayer/gui/main_frame'
+require 'iplayer/version'
+
+options = {
+  :type_preference  => %w[original signed],
+  :http_proxy       => ENV['http_proxy']
+}
+about = {
+  :name        => 'iPlayer Downloader',
+  :version     => "#{IPlayer::GUI_VERSION} (library #{IPlayer::VERSION})",
+  :developers  => ['Paul Battley'],
+  :description => "Download programmes from the BBC iPlayer.\nVisit http://po-ru.com/projects/iplayer-downloader/\nfor more information."
+}
+app = IPlayer::GUI::App.new(IPlayer::GUI::MainFrame, about, options)
+app.main_loop

data/lib/iplayer.rb

@@ -0,0 +1,5 @@
+require 'iplayer/errors'
+require 'iplayer/browser'
+require 'iplayer/metadata'
+require 'iplayer/subtitles'
+require 'iplayer/downloader'

data/lib/iplayer/browser.rb

@@ -0,0 +1,69 @@
+require 'net/http'
+require 'uri'
+
+class Net::HTTPResponse # Monkey-patch in some 21st-century functionality
+  include Enumerable
+  
+  def cookies
+    inject([]){ |acc, (key, value)|
+      key == 'set-cookie' ? acc << value.split(/;/).first : acc
+    }
+  end
+  
+  def to_hash
+    @to_hash ||= inject({}){ |hash, (key, value)|
+      hash[key] = value
+      hash
+    }
+  end
+end
+
+module IPlayer
+class Browser
+
+  # Used by Safari Mobile
+  IPHONE_UA  = 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ '+
+               '(KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3'
+
+  # Used by Quicktime
+  QT_UA      = 'Apple iPhone v1.1.4 CoreMedia v1.0.0.4A102'
+
+  # Safari, for no good reason
+  DESKTOP_UA = 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_4; en-gb) '+
+               'AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.20.1'
+
+  DEFAULT_HEADERS = {
+    'Accept'          => '*/*',
+    'Accept-Language' => 'en',
+    'Connection'      => 'keep-alive',
+    'Pragma'          => 'no-cache'
+  }
+
+  def initialize(http_class = Net::HTTP)
+    @http_class = http_class
+  end
+
+  def get(location, headers={}, &blk)
+    url = URI.parse(location)
+    http = @http_class.new(url.host, url.port)
+    path = url.path
+    if url.query
+      path << '?' << url.query
+    end
+    if defined? DEBUG
+      puts path
+      DEFAULT_HEADERS.merge(headers).each do |k,v|
+        puts " -> #{k}: #{v}"
+      end
+    end
+    response = http.request_get(path, DEFAULT_HEADERS.merge(headers), &blk)
+    if defined? DEBUG
+      response.each do |k,v|
+        puts "<-  #{k}: #{v}"
+      end
+    end
+    response
+  end
+
+end
+end

data/lib/iplayer/downloader.rb

@@ -0,0 +1,132 @@
+require 'tempfile'
+
+module IPlayer
+class Downloader
+
+  IPHONE_URL     = 'http://www.bbc.co.uk/mobile/iplayer/'
+  SELECTOR_URL   = 'http://www.bbc.co.uk/mediaselector/3/auth/iplayer_streaming_http_mp4/%s?%s'
+  BUG_URL        = 'http://www.bbc.co.uk/iplayer/framework/img/o.gif?%d'
+  MAX_SEGMENT    = 4 * 1024 * 1024
+  COPY_BUFFER    = 4 * 1024 * 1024
+
+  include IPlayer::Errors
+
+  Version = Struct.new(:name, :pid)
+
+  attr_reader   :browser, :pid
+  attr_accessor :cookies
+
+  def self.extract_pid(pid_or_url)
+    case pid_or_url
+    when %r!/(?:item|episode|programmes)/([a-z0-9]{8})!
+      $1
+    when %r!^[a-z0-9]{8}$!
+      pid_or_url
+    when %r!(b0[a-z0-9]{6})!
+      $1
+    else
+      raise NotAPid, pid_or_url
+    end
+  end
+
+  def initialize(browser, pid)
+    @browser = browser
+    @pid = pid
+  end
+
+  def metadata
+    @metadata = Metadata.new(@pid, @browser)
+  end
+
+  def get(url, user_agent, options={}, &blk)
+    options['User-Agent'] = user_agent
+    options['Cookie'] = cookies if cookies
+    browser.get(url, options, &blk)
+  end
+  
+  def available_versions
+    metadata.versions.map{ |name, vpid| Version.new(name, vpid) }
+  end
+
+  def download(version_pid, path, options={}, &blk)
+    if options[:subtitles]
+      download_subtitles(version_pid, path)
+    end
+
+    if File.exist?(path)
+      offset = File.size(path)
+    else
+      offset = 0
+    end
+    
+    File.open(path, 'a+b') do |io|    
+      location = real_stream_location(version_pid)
+      content_length = content_length_from_initial_request(location)
+      yield(offset, content_length) if block_given?
+      
+      offset.step(content_length - 1, MAX_SEGMENT) do |first_byte|
+        last_byte = [first_byte + MAX_SEGMENT - 1, content_length - 1].min
+        get(location, Browser::QT_UA, 'Range'=>"bytes=#{first_byte}-#{last_byte}") do |response|
+          response.read_body do |data|
+            offset += data.length
+            io << data
+            yield(offset, content_length) if block_given?
+          end
+        end
+      end
+    end
+  end
+
+private
+
+  def request_iphone_page
+    response = get(IPHONE_URL, Browser::IPHONE_UA)
+    raise ProgrammeDoesNotExist unless response.is_a?(Net::HTTPSuccess)
+    self.cookies = response.cookies.join('; ')
+  end
+
+  def request_image_bugs
+    get(BUG_URL % [(rand * 100000).floor], Browser::IPHONE_UA)
+  end
+
+  def real_stream_location(version_pid)
+    request_iphone_page
+    request_image_bugs
+
+    # Get the auth URL
+    r = (rand * 10000000).floor
+    selector = SELECTOR_URL % [version_pid, r]
+    response = get(selector, Browser::QT_UA, 'Range'=>'bytes=0-1')
+    
+    # It redirects us to the real stream location
+    location = response.to_hash['location']
+    if location =~ /error\.shtml/
+      raise FileUnavailable
+    end
+    return location
+  end
+
+  def content_length_from_initial_request(location)
+    # The first request of CoreMedia is always for the first byte
+    response = get(location, Browser::QT_UA, 'Range'=>'bytes=0-1')
+
+    # We now know the full length of the content
+    content_range = response.to_hash['content-range']
+    unless content_range
+      raise FileUnavailable
+    end
+    return content_range[/\d+$/].to_i
+  end
+
+  def download_subtitles(version_pid, media_path)
+    subtitles = Subtitles.new(version_pid, browser)
+    xml = subtitles.w3c_timed_text
+    return if xml.nil?
+    subtitles_path = media_path.sub(/\.[^\.]+$/, '.xml')
+    return if File.exist?(subtitles_path)
+    File.open(subtitles_path, 'w') do |f|
+      f << xml
+    end
+  end
+end
+end