irecorder 0.0.4-linux

data/lib/bbcnet.rb

@@ -0,0 +1,244 @@
+#
+#
+#
+require 'rubygems'
+require 'uri'
+require 'net/http'
+require 'open-uri'
+require 'rss'
+require 'nokogiri'
+require 'shellwords'
+require 'fileutils'
+
+UrlRegexp = URI.regexp(['rtsp','http'])
+
+#
+#
+class BBCNet
+    RtspRegexp = URI.regexp(['rtsp'])
+    MmsRegexp = URI.regexp(['mms'])
+    DirectStreamRegexp = URI.regexp(['mms', 'rtsp', 'rtmp', 'rtmpt'])
+
+
+    #------------------------------------------------------------------------
+    # get stream metadata
+    # episode url => pid => xml playlist => version pid (vpid aka. identifier)
+    #   => xml stream metadata => wma
+    #
+    class MetaInfo
+        def self.get(url)
+            pid = BBCNet.extractPid(url)
+#             info = @@cachePid[pid]
+#             return info if info
+            self.new(pid)
+        end
+
+        attr_reader :pid
+        Keys = [ :duration, :vpid, :group, :media, :onAirDate, :channel, :title, :summary, :aacLow, :aacStd, :real, :wma, :streams ]
+        def initialize(pid)
+            @pid = pid
+            Keys.each do |k|
+                s = ('@' + k.to_s).to_sym
+                self.instance_variable_set(s, nil)
+                self.class.class_eval %Q{
+                    def #{k}
+                        #{s}
+                    end
+                }
+            end
+
+            @streams = []
+        end
+
+
+        #
+        # read duration, vpid, group, media, onAirDate, channel
+        #   from XmlPlaylist
+        def readXmlPlaylist
+            return self if @vpid
+
+            res = BBCNet.read("http://www.bbc.co.uk/iplayer/playlist/#{@pid}")
+#             res = IO.read("../tmp/iplayer-playlist-me.xml")
+
+            doc = Nokogiri::XML(res)
+            item = doc.at_css("item")
+            @media = item[:kind].gsub(/programme/i, '')
+            @duration = item[:duration].to_i
+            @vpid = item[:identifier]
+            @group = item[:group]
+            @onAirDate = BBCNet.getTime(item.at_css("broadcast").content.to_s)
+            @channel = item.at_css("service").content.to_s
+            @title = item.at_css("title").content.to_s
+            @summary = doc.at_css("summary").content.to_s
+            self
+        end
+
+
+        class StreamInfo
+            #  example)    48, wma, time, audio, http://..
+            attr_accessor :bitrate, :encoding, :expires, :type, :indirectUrl
+            alias       :kind :type
+
+            def url
+                @url ||= BBCNet.getDirectStreamUrl(@indirectUrl)
+            end
+        end
+
+        def readXmlStreamMeta
+            readXmlPlaylist unless @vpid
+
+            res = BBCNet.read("http://www.bbc.co.uk/mediaselector/4/mtis/stream/#{vpid}")
+#             res = IO.read("../tmp/iplayer-stream-meta-me.xml")
+
+            doc = Nokogiri::XML(res)
+            me = doc.css("media")
+            me.each do |m|
+                stmInf = StreamInfo.new
+                stmInf.encoding = m[:encoding]  # wma
+                stmInf.bitrate = m[:bitrate].to_i    # 48
+                expiresStr = m[:expires]
+                stmInf.expires = BBCNet.getTime(expiresStr)  if expiresStr
+                stmInf.type = m[:kind]          # audio
+
+                con = m.at_css("connection")
+                stmInf.indirectUrl = con[:href]
+                @streams <<= stmInf
+
+                case stmInf.encoding
+                when /\bwma\b/i
+                    @wma = stmInf
+                when /\baac\b/i
+                    if stmInf.bitrate < 64
+                        @aacLow = stmInf
+                    else
+                        @aacStd = stmInf
+                    end
+                when /\breal\b/i
+                    @real = stmInf
+                end
+            end
+            self
+        end
+
+        alias :update :readXmlStreamMeta
+
+    end
+
+
+
+    def self.getTime(str)
+        tm = str.match(/(\d{4})-(\d\d)-(\d\d)\w(\d\d):(\d\d):(\d\d)/)
+        par = ((1..6).inject([]) do |a, n| a << tm[n].to_i end)
+        Time.gm( *par )
+    end
+
+
+    #------------------------------------------------------------------------
+    #
+    #
+
+    # convert epsode Url to console Url
+    # example
+    #  from
+    #    http://www.bbc.co.uk/iplayer/episode/b007jpkt/Miss_Marple_A_Caribbean_Mystery_Episode_1/
+    #  to
+    #    http://www.bbc.co.uk/iplayer/console/b007jpkt
+    def self.getPlayerConsoleUrl(url)
+       "http://www.bbc.co.uk/iplayer/console/" + extractPid(url)
+    end
+
+    # get PID from BBC episode Url
+    def self.extractPid(url)
+        case url
+        when %r!/(?:item|episode|programmes)/([a-z0-9]{8})!
+            $1
+        when %r!^[a-z0-9]{8}$!
+            url
+        when %r!\b(b[a-z0-9]{7}\b)!
+            $1
+        else
+            raise "No PID in Url '%s'" % url
+        end
+    end
+
+
+    # .asf/.ram => .wma/.ra
+    def self.getDirectStreamUrl(url)
+        unless url[DirectStreamRegexp] then
+            res = self.read(url)
+            newSrc= res[ DirectStreamRegexp ]
+            url = newSrc unless newSrc.nil?
+        end
+        url
+    end
+
+
+    def self.read(url)
+        header = { "User-Agent" => self.randomUserAgent }
+        if defined? @@proxy
+            header[:proxy] = @@proxy
+        end
+
+        uri = URI.parse(url)
+        res = Net::HTTP.start(uri.host, uri.port) do |http|
+            http.get(uri.path, header)
+        end
+        res.body
+    end
+
+    def self.setProxy(url)
+        @@proxy = url
+    end
+
+    private
+    UserAgentList = [
+        'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/<RAND>.8 (KHTML, like Gecko) Chrome/2.0.178.0 Safari/<RAND>.8',
+        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; YPC 3.2.0; SLCC1; .NET CLR 2.0.50<RAND>; .NET CLR 3.0.04<RAND>)',
+        'Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; tr) AppleWebKit/<RAND>.4+ (KHTML, like Gecko) Version/4.0dp1 Safari/<RAND>.11.2',
+        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50<RAND>; .NET CLR 3.5.30<RAND>; .NET CLR 3.0.30<RAND>; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)',
+        'Mozilla/6.0 (Windows; U; Windows NT 7.0; en-US; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.9 (.NET CLR 3.5.30<RAND>)',
+        ]
+    def self.randomUserAgent
+        ua = UserAgentList[ rand UserAgentList.length ]
+        ua.gsub(/<RAND>/, "%03d" % rand(1000))
+    end
+
+end
+
+module AudioFile
+    # return seconds of audio file duration.
+    def self.getDuration(file)
+        case file[/\.\w+$/].downcase
+        when ".mp3"
+            cmd = "| exiftool -S -Duration %s" % file.shellescape
+        when ".wma"
+            cmd = "| exiftool -S -PlayDuration %s" % file.shellescape
+        end
+        msg = open(cmd) do |f| f.read end
+        a = msg.scan(/(?:(\d+):){0,2}(\d+)/)[0]
+        i = -1
+        a.reverse.inject(0) do |s, d|
+            i += 1
+            s + d.to_i * [ 1, 60, 3600 ][i]
+        end
+    end
+end
+
+
+
+if __FILE__ == $0 then
+    puts AudioFile::getDuration(ARGV.shift)
+    exit 0
+
+    pid = "b00mzvfq"
+    if ARGV.size > 0 then
+        pid = ARGV.shift
+    end
+    minfo = BBCNet::MetaInfo.new(pid)
+    minfo.readXmlStreamMeta
+    puts minfo.inspect
+
+    minfo.streams.each do |s|
+        puts "url : " + s.url
+    end
+end

data/lib/download.rb

@@ -0,0 +1,325 @@
+require 'fileutils'
+
+require "bbcnet.rb"
+
+#-------------------------------------------------------------------
+#
+#
+#
+class DownloadProcess < Qt::Process
+    slots   'taskFinished(int,QProcess::ExitStatus)'
+    attr_reader     :sourceUrl, :fileName
+    attr_accessor   :taskItem
+
+    #
+    DEBUG_DOWNLOAD = false
+
+    # @stage
+    DOWNLOAD = 0
+    CONVERT = 1
+    FINISHED = 2
+
+    # @status
+    INITIAL = 0
+    RUNNING = 1
+    ERROR = 2
+    DONE = 3
+    def statusMessage
+        case @status
+        when INITIAL, RUNNING, DONE
+            %w{ Downloading Converting Finished }[@stage]
+        when ERROR
+            "Error : " + %w{ Download Convert Finish }[@stage]
+        else
+            "???"
+        end
+    end
+
+    def status=(st)
+        @status = st
+        @taskItem.status = statusMessage if @taskItem
+        $log.misc { "status:#{status}" }
+    end
+
+    #--------------------------
+    # check status
+    def running?
+        @status == RUNNING
+    end
+
+    def error?
+        @status == ERROR
+    end
+
+    #--------------------------
+    # check stage
+    def finished?
+        @stage == FINISHED
+    end
+
+    def rawDownloaded?
+        @stage >= CONVERT
+    end
+
+    class Command
+        attr_accessor   :app, :args, :msg
+        def initialize(app, args, msg)
+            @app = app
+            @args = args
+            @msg = msg
+        end
+    end
+
+    attr_reader :sourceUrl, :rawFileName
+
+    def initialize(parent, metaInfo, fName)
+        super(parent)
+        @metaInfo = metaInfo
+        @parent = parent
+        @taskItem = nil
+        @startTime = Time.new
+        @sourceUrl = @metaInfo.wma.url
+        @rawFileName = fName
+        @rawFilePath = File.join(IRecSettings.rawDownloadDir.path, fName)
+        mkdirSavePath(@rawFilePath)
+        $log.debug { "@rawFilePath : #{@rawFilePath }" }
+
+        @stage = DOWNLOAD
+        @status = INITIAL
+
+        connect(self, SIGNAL('finished(int,QProcess::ExitStatus)'), self, SLOT('taskFinished(int,QProcess::ExitStatus)') )
+    end
+
+
+    def beginTask
+        # 1st stage : download
+        if File.exist?(@rawFilePath) then
+            @stage = CONVERT
+            @outFileName = @rawFileName.gsub(/\.\w+$/i, '.mp3')
+            @outFilePath = File.join(IRecSettings.downloadDir.path, @outFileName)
+            $log.debug { "@rawFilePath duration:" + AudioFile.getDuration(@rawFilePath).to_s }
+            $log.debug { "@outFilePath duration:" + AudioFile.getDuration(@outFilePath).to_s }
+            $log.debug { "metainf duration:" + @metaInfo.duration.to_s }
+            if File.exist?(@outFilePath) then
+                taskFinished(1,0)
+            else
+                $log.debug { "begin convert" }
+                beginConvert
+                return
+            end
+        else
+            beginDownload
+        end
+
+#         beginDownload
+    end
+
+
+    def retryTask
+        $log.debug { "retry! in main." }
+        if error? then
+            # retry
+            case @stage
+            when DOWNLOAD
+                @startTime = Time.new
+                beginDownload
+            when CONVERT
+                @startTime = Time.new
+                beginConvert
+            end
+        else
+            $log.warn { "cannot retry the successfully finished or running process." }
+        end
+    end
+
+    def cancelTask
+        if running? then
+            self.terminate
+        end
+    end
+
+    def updateLapse
+        taskItem.updateTime(lapse)
+    end
+
+    def lapse
+        Time.now - @startTime
+    end
+
+    # slot :
+    def taskFinished(exitCode, exitStatus)
+        checkReadOutput
+        if (exitCode.to_i.nonzero? || exitStatus.to_i.nonzero?) && checkErroredStatus then
+            self.status = ERROR
+            $log.error { [ makeErrorMsg, "exitCode=#{exitCode}, exitStatus=#{exitStatus}" ] }
+        else
+            $log.info {
+                [ "Successed to download a File '%#2$s'",
+                    "Successed to convert a File '%#2$s'", ][@stage] %
+                [ @sourceUrl, @rawFilePath ]
+            }
+            nextTask
+        end
+    end
+
+    def updateView
+        if running? then
+            # update Lapse time
+            updateLapse
+
+            # dump IO message buffer
+            checkReadOutput
+        end
+    end
+
+
+    protected
+    # increment stage
+    def nextTask
+        @stage += 1
+        case @stage
+        when DOWNLOAD
+            beginDownload
+        when CONVERT
+            beginConvert
+        else
+            removeRawFile
+            allTaskFinished
+        end
+    end
+
+    def beginDownload
+        $log.info { " DownloadProcess : beginDownload." }
+        @stage = DOWNLOAD
+        @downNG = true
+        self.status = RUNNING
+        @currentCommand = makeMPlayerDownloadCmd
+
+        $log.info { @currentCommand.msg }
+        start(@currentCommand.app, @currentCommand.args)
+    end
+
+    def makeMPlayerDownloadCmd
+        # make MPlayer Downlaod comand
+        cmdMsg = "mplayer -noframedrop -dumpfile %s -dumpstream %s" %
+                    [@rawFilePath.shellescape, @sourceUrl.shellescape]
+        cmdApp = "mplayer"
+        cmdArgs = ['-noframedrop', '-dumpfile', @rawFilePath, '-dumpstream', @sourceUrl]
+
+        # debug code.
+        if DEBUG_DOWNLOAD then
+            if rand > 0.4 then
+                cmdApp = "test/sleepjob.rb"
+                cmdArgs = %w{ touch a/b/ }
+            else
+                cmdApp = "test/sleepjob.rb"
+                cmdArgs = %w{ touch } << @rawFilePath.shellescape
+            end
+        end
+
+        Command.new( cmdApp, cmdArgs, cmdMsg )
+    end
+
+    def beginConvert
+        @stage = CONVERT
+        self.status = RUNNING
+        @outFileName = @rawFileName.gsub(/\.\w+$/i, '.mp3')
+        @outFilePath = File.join(IRecSettings.downloadDir.path, @outFileName)
+        mkdirSavePath(@outFilePath)
+
+        cmdMsg = "nice -n 19 ffmpeg -i %s -f mp3 %s" %
+                    [ @rawFilePath.shellescape, @outFilePath.shellescape]
+        cmdApp = "nice"
+        cmdArgs = [ '-n', '19', 'ffmpeg', '-i', @rawFilePath, '-f', 'mp3', @outFilePath ]
+
+
+        # debug code.
+        if DEBUG_DOWNLOAD then
+            if rand > 0.4 then
+                cmdApp = "test/sleepjob.rb"
+                cmdArgs = %w{ touch a/b/ }
+            else
+                cmdApp = "test/sleepjob.rb"
+                cmdArgs = %w{ cp -f } + [ @rawFilePath.shellescape, @outFilePath.shellescape ]
+            end
+        end
+
+        @currentCommand = Command.new( cmdApp, cmdArgs, cmdMsg )
+        $log.info { @currentCommand.msg }
+        start(@currentCommand.app, @currentCommand.args)
+    end
+
+
+    def removeRawFile
+        unless IRecSettings.leaveRawFile then
+            begin
+                File.delete(@rawFilePath)
+            rescue => e
+                $log.error { e }
+            end
+        end
+    end
+
+    def checkOutput(msg)
+        msgSum = msg.join(' ')
+        @downNG &&= false if msgSum =~ /Everything done/i
+    end
+
+    # check and read output
+    def checkReadOutput
+        msg = readAllStandardOutput.data .reject do |l| l.empty? end
+        checkOutput(msg)
+        $log.info { msg }
+    end
+
+    # return error or not
+    def checkErroredStatus
+        case @stage
+        when DOWNLOAD
+            return @downNG if @downNG
+            begin
+                $log.debug { "check duration for download." }
+                AudioFile.getDuration(@rawFilePath) < @metaInfo.duration - 5
+            rescue => e
+                $log.info{ e }
+                true
+            end
+        when CONVERT
+            begin
+                $log.debug { "check duration for convert." }
+                AudioFile.getDuration(@outFilePath) < @metaInfo.duration - 40
+            rescue => e
+                $log.info{ e }
+                true
+            end
+        else
+            true
+        end
+    end
+
+
+
+
+    protected
+    def allTaskFinished
+        @stage = FINISHED
+        self.status = DONE
+    end
+
+    def makeErrorMsg
+        [ "Failed to download a File '%#2$s'",
+            "Failed to convert a File '%#2$s'", ][@stage] %
+            [ @sourceUrl, @rawFilePath ]
+    end
+
+
+    def mkdirSavePath(fName)
+        dir = File.dirname(fName)
+        unless File.exist? dir
+            $log.info{ "mkdir : " +  dir }
+            FileUtils.mkdir_p(dir)
+        end
+    end
+end
+
+