xmltv 0.8.1

data/lib/xmltv/sites/rt.rb

@@ -0,0 +1,133 @@
+#!/usr/bin/ruby -w
+
+autoload :XMLTV, 'xmltv/xmltv'
+
+
+module XMLTV
+
+  class RtGrabber < Grabber
+    Fields = %w{ 
+      title sub_title episode date director cast
+      premiere film repeat subtitles widescreen
+      new_series deaf_signed black_and_white star_rating
+      certificate category desc choice datum start stop
+      duration_mins
+    }
+    TimeDiff = 60 * 60
+    EncodingErrors = {
+      /\303[\306\346\341]/ => "'",
+      /\310\355/ => "\303\241",
+      /\310\321/ => "\303\245"
+    }
+    def version
+      [ "RtGrabber 0.9", super].join("\n")
+    end
+
+    def base_url
+      'http://xmltv.radiotimes.com/xmltv'
+    end
+
+    def fetch_all_channels
+      chn = open("#{base_url}/channels.dat") {|h| h.readlines }
+      channels = Hash.new
+      chn.each do |c|
+        chan_id, name = c.chomp.split('|')
+        channels[chan_id] = name
+      end
+      save_object(channels, channel_list)
+      channels
+    end
+
+    def lang
+      'en'
+    end
+
+
+  ###  Must Implement
+
+
+    def grab_channel(chan_id)
+      uri = "#{base_url}/#{chan_id}.dat"
+  #    uri = '/home/han/47.dat'
+  #    uri = '/var/tmp/xmltv/102.dat'
+      info = open(uri) {|h| h.readlines }.map{|x| x.chomp}
+      @hits += 1
+      @channel_info = Array.new
+      info.each do |program|
+        begin
+          prog = program.split('~')
+          entry = Hash.new
+          Fields.each_with_index do |el, i |
+            item = prog[i]
+            next if item == 'false' || item.empty?
+            begin
+              entry[el] = item.ck_utf
+            rescue Iconv::IllegalSequence => e
+              ret = false
+              EncodingErrors.each_pair do |k, v|
+  #              STDERR.puts "Try: #{k.dump} #{v}\n#{item.dump}\n"
+                if item.gsub!(k, v)
+  #                STDERR.puts "====> #{item.dump}\n"
+                  ret = true
+                  break
+                end
+  #              STDERR.puts 'failed'
+              end
+              retry if ret
+              prt = (e.failed =~ /[[:print:]]/)
+              failstring = e.failed[0 ... prt]
+              STDERR.puts "Repl: #{failstring.dump}"
+              EncodingErrors[failstring] = '?'
+              retry if item.gsub!(e.failed[0 ... prt], '?') 
+              raise "Encoding error: #{e.failed}\n"
+            end 
+
+          end
+    #      puts entry['cast'], entry['start'], entry['stop']
+           @channel_info << entry
+        rescue StandardError => exc
+          STDERR.puts exc.class, exc.message, exc.backtrace
+          PP.pp prog, STDERR
+          raise
+        end
+      end
+      info.size
+    end
+
+    def transform(chan_id)
+      progdata_array = Array.new
+      @channel_info.each do |p|
+        progdata_array << ( progdata = proghash(p, chan_id))
+        dag, maand, jaar = p['datum'].scan(/\d+/).map{|x| x.to_i}
+        startuur, startmin = p['start'].scan(/\d+/).map{|x| x.to_i}
+        stopuur, stopmin = p['stop'].scan(/\d+/).map{|x| x.to_i}
+        progdata['start']  = Time.local(jaar, maand, dag, startuur, startmin) + TimeDiff
+        date_stats(chan_id, progdata['start'])
+        progdata['stop'] = Time.local(jaar, maand, dag, stopuur, stopmin) + TimeDiff
+        progdata['stop'] += Dag if progdata['stop'] < progdata['start']
+        progdata['credits']['director'] = p['director'] if p['director']
+        progdata['video']['colour'] = 'no' if p['black_and_white']
+        progdata['video']['aspect'] = '16:9' if p['widescreen']
+        progdata['sub-title'] = p['episode'] || p['sub_title'] || ''
+        progdata['category'] = 'Film' if p['film']
+        progdata['subtitles']['type'] = 'teletext' if p['subtitles']
+        progdata['star-rating']['value'] = "#{p['star_rating']}/5" if p['star_rating']
+        if (cast = p['cast'])
+          progdata['credits']['actor'] = 
+            cast.index('|') ?
+              cast.split('|').map {|x| x.gsub(/^.*[*]/,'') } :
+              cast.split(',')
+        end
+        %w{ widescreen black_and_white }.each do |it|
+  #        STDERR.puts "#{it}: #{p[it]}"
+        end
+       # pp progdata
+       # pp p
+
+      end
+      progdata_array
+    end
+  end
+end
+XMLTV::RtGrabber.new.run
+

data/lib/xmltv/sites/trivial.rb

@@ -0,0 +1,33 @@
+#!/usr/bin/ruby -w
+
+autoload :XMLTV, 'xmltv/xmltv'
+
+
+module XMLTV
+
+  class TrivialGrabber < Grabber
+    # Must implement fetch_all_channels or define channel_list as nil
+    # Must return something that respond_to? []
+    def fetch_all_channels
+      { "1" => "Channel" }
+    end
+    # grab_channel returns number of programs found
+    def grab_channel(chan_id)
+      1
+    end
+    # transform returns an array of special hashes (obtained with proghash)
+    # 
+    def transform(chan_id)
+      progdata_array = Array.new
+      progdata = proghash({}, chan_id)
+      progdata['start'] = Time.new
+      progdata['stop'] = Time.new + 60 * 60
+      progdata['title'] = 'Title'
+      progdata_array << progdata
+        
+      progdata_array
+    end
+  end
+
+end
+XMLTV::TrivialGrabber.new.run

data/lib/xmltv/sites/tvgids.rb

@@ -0,0 +1,224 @@
+#!/usr/bin/ruby -w
+
+autoload :XMLTV, 'xmltv/xmltv'
+require 'cgi'
+
+
+
+module XMLTV
+
+  class TvgidsGrabber < Grabber
+    Cattrans = { 
+      'amusement'        => 'Talk',
+      'animatie'         => 'Animated',
+      'comedy'           => 'Comedy',
+      'documentaire'     => 'Documentary',
+      'educatief'        => 'Educational',
+      'erotiek'          => 'Adult',
+      'film'             => 'Movies',
+      'muziek'           => 'Art/Music',
+      'informatief'      => 'Educational',
+      'jeugd'            => 'Children',
+      'kunst/cultuur'    => 'Arts/Culture',
+      'misdaad'          => 'Crime/Mystery',
+      'muziek'           => 'Music',
+      'natuur'           => 'Science/Nature',
+      'nieuws/actualiteiten' => 'News',
+      'overige'          => 'Unknown',
+      'religieus'        => 'Religion',
+      'serie/soap'       => 'Drama',
+      'sport'            => 'Sports',
+      'theater'          => 'Arts/Culture',
+      'wetenschap'       => 'Science/Nature'
+    }
+    Roletrans = {
+      'regie'             => 'director',
+      'acteurs'           => 'actor',
+      'presentatie'       => 'presenter',
+      'scenario'          => 'writer'
+    }
+
+    Titeltrans = {
+      'titel aflevering' => 'sub-title',
+      'jaar van premiere' => 'date',
+      'aflevering' => 'episode-num'
+    }
+
+    def grab_detail(href)
+      if href[0] == ?/
+        href="#{base_url}#{href}"
+      end
+      STDERR.puts "#{Time.now}: #{href} #{@channelhash.size}" if XmltvOptions.verbose
+      program = Hash.new
+      details = fetch(href)
+      desc = []
+      details.at('//table#progDetail').search('//tr//p').each do |p|
+        break if p['class'] == 'meerLinks'
+        line = p.inner_text.strip
+        desc << line unless line.empty?
+      end
+      program['desc'] = desc.join(' ').to_utf
+      details.search('//div#progPropt//tr/th').each do |pg|
+        content = pg.at('../td')
+        if content['class'] == 'personen'
+          rsl = content.at('div').search('.').find_all { |x| x.text? }.map{|x| x.to_s.strip.to_utf}.find_all{|x| ! x.empty?}
+        else
+          rsl = content.inner_text.strip.to_utf
+        end
+        program[pg.inner_text.strip.gsub(':','').downcase] = rsl 
+      end
+  #    PP.pp program, STDERR
+      program
+    end
+
+    def channel_url(chan_id)
+      "#{base_url}//zoeken/?periode=9&station=#{chan_id}"
+    end
+
+    def fetch_all_channels
+      page = fetch(channel_url(1))
+      channels = Hash.new
+      page.search('//optgroup')[0..1].each do |og|
+        og.search('/option').each do |g|
+          channels[g['value']] = g.inner_text
+        end
+      end
+      save_object(channels, channel_list)
+      channels
+    end
+
+    def clean_cache(cache)
+      count = 0
+      cache.delete_if do |dt, en|
+        rsl = (Date.dutch(en['datum']) < Vandaag)
+  #      puts Date.dutch(en['datum']), Vandaag, rsl, '==='
+        count += 1 if rsl
+        rsl
+      end
+      count
+    end
+
+    def grab_channel(chan_id)
+      url = channel_url(chan_id)
+      page = fetch(url)
+      @channelhash = load_cachefile(chan_id)
+  #    get_file(chan_id)
+      period = datum = nil
+      fetched = 0
+      begin
+        found = remaining = page.at("//table.overzicht//tr//td/strong").inner_text.to_i
+      rescue NoMethodError
+        niks =  page.at("//div#resultaten").at("//td").inner_text
+        STDERR.puts url, niks
+        return
+      end
+  #    STDERR.puts("#{found} on site, #{@channelhash.size} in cache")
+      page.search("//table.overzicht//tr").each do |pg|
+        td = pg.at('td')
+        next if td.nil? || pg['class'] == 'zoekstring'
+        if td['class'] == 'bloktitel'
+          period = td.at('h5').inner_text rescue period
+          datum = td.at('h4').inner_text rescue datum
+          next
+        end
+  #      puts "Period: #{period} "
+        if (tijd = pg.at('/th').inner_text) =~ /\d\d:\d\d/
+          det =  pg.at('/td//a')
+          href = det['href']
+          id = href[/ID=(\d+)/,1]
+  #        puts @channelhash[id]
+          remaining -= 1
+          next if @channelhash[id]
+          fetched += 1
+          begin
+            @channelhash[id] = program = grab_detail(href)
+          rescue
+            STDERR.puts href, pg, '====='
+            raise
+          end
+          program['title'] = det.inner_text.strip.to_utf
+          program['period'] = period
+          program['datum'] = datum
+          program['tijd'] = tijd
+          program['progtip'] = '4/5' if pg['class'] == 'progTip'
+        end
+      end
+      STDERR.puts "Something wrong remaining: #{remaining}" if remaining != 0
+      save_object(@channelhash, cachefile(chan_id)) if fetched > 0
+      found
+    end
+
+    def parse_times(str)
+      rsl = nil
+      md = /(\d+)\s(\w+)\s(\d+),\s(\d+):(\d+)/.match(str)
+      if md
+        rsl = md.captures.map do |x|
+            x =~ /\d/ ? x.to_i : Date::Maanden.index(x)
+        end
+        mdstop = /(\d+):(\d+)/.match(str[md.offset(0)[1]..-1])
+        if mdstop
+          rsl << mdstop.captures.map {|x| x.to_i}
+        end
+        rsl.flatten!
+      end
+      rsl
+    end
+    def transform(chan_id)
+  #    get_file(chan_id)
+  #    STDERR.print "#{chan_id} #{@channelhash.size}"
+      progdata_array = Array.new
+      @channelhash.each_pair do |id, entry|
+        begin
+          progdata = proghash(entry, chan_id)
+          a=entry['datum en tijdstip']
+          shift = entry['period'] == 'Nacht' ? Dag : 0
+          dag, maand, jaar, startuur, startmin, stopuur, stopmin = parse_times(a)
+          next if dag.nil?
+          progdata['start'] = start = Time.local(jaar, maand, dag, startuur, startmin) + shift
+          
+          if stopuur
+            stop = Time.local(jaar, maand, dag, stopuur, stopmin) + shift
+            if start > stop && start.hour >= 21 && stop.hour <= 7
+              stop += Dag
+            end
+            progdata['stop'] = stop
+          end
+          date_stats(chan_id, progdata['start'])
+          if (b = entry['bijzonderheden'])
+            b.downcase.split(',').each do |bijz|
+              case bijz
+                when /breedbeeld/
+                  progdata['video']['aspect'] = '16:9'
+                when /zwart/
+                  progdata['video']['colour'] = 'no'
+               when /teletekst/
+                  progdata['subtitles']['type'] = 'teletext'
+                when /stereo/
+                  progdata['audio']['stereo'] = 'stereo'
+              end
+            end
+          end
+          %w{ regie acteurs scenario presentatie }.each do |role|
+            if entry[role]
+              progdata['credits'][Roletrans[role]] = entry[role]
+            end
+          end
+          progdata['category'] = Cattrans[entry['genre'].downcase] if entry['genre']
+          progdata['star-rating']['value'] = entry['progtip'] if entry['progtip']
+          Titeltrans.each do |key|
+            progdata[Titeltrans[key]] = entry[key] if entry[key]
+          end
+          progdata_array << progdata
+
+        rescue StandardError => exc
+          STDERR.puts exc, exc.message, exc.backtrace
+          PP.pp(entry, STDERR)
+          raise
+        end
+      end
+      progdata_array
+    end
+  end
+  
+end
+XMLTV::TvgidsGrabber.new.run

data/lib/xmltv/sites/tvtoday.rb

@@ -0,0 +1,185 @@
+#!/usr/bin/ruby -w
+
+autoload :XMLTV, 'xmltv/xmltv'
+require 'date'
+
+class Date
+  def german
+    "#{day}.#{month}.#{year}"
+  end
+end
+
+module XMLTV
+  class TvtodayGrabber < Grabber
+    # Must implement fetch_all_channels or define channel_list as nil
+    # Must returns something that respond_to? []
+    def lang
+      'de'
+    end     
+    def fetch_all_channels
+      puts base_url
+      url = @base_url
+      page = fetch(url)
+      channels = Hash.new
+      page.at('//select[@name=channel]').search('//option').each do |op|
+        val = op['value']
+        next if val.index('gruppeId') || val.empty?
+        channels[val] = op.inner_text.strip.to_utf
+      end
+      save_object(channels, channel_list)
+      channels
+    end
+    def grab_detail(href)
+      program = Hash.new
+      if href[0] == ?/
+        href="#{base_url}#{href}"
+      end
+#      puts href
+      page = fetch(href)
+      info = page.at('span.tv-titel-popup').parent
+#      puts info.at('span.tv-titel-popup').inner_text.to_utf
+      program['desc-s'] = info.at('span.tv-vorspann-popup').inner_text.to_utf
+      page.at('td.tv-sendung-info').each_child do |el|
+        if el.is_a? Hpricot::Text
+          program['desc'] = el.inner_text.to_utf
+          break
+        end
+      end
+      credits = page.at('span.tv-credits')
+      if credits
+        program['info'] = credits.parent.inner_text.to_utf
+      end
+      categ = info.at('span.tv-auszeichnung')
+      if categ
+        program['kind'] = categ.inner_text.to_utf
+      end
+      info.search('img').each do |img|
+        program[img['title']] = true
+      end
+      program
+    end
+    
+    def clean_cache(cache)
+      count = 0
+      cache.delete_if do |dt, en|
+        rsl = (Date.parse(en['datum']) < Vandaag)
+        count += 1 if rsl
+        rsl
+      end
+      count
+    end
+    
+    def day_url(chan_id, day)
+      "#{base_url}/program2007?format=genre&offset=0&date=#{day.german}&slotIndex=all&channel=#{chan_id}&order=time"
+    end
+    # grab_channel return chan_idber of programs found
+    def grab_channel(chan_id)
+      fetched = found = 0
+      @channelhash = load_cachefile(chan_id)
+      now = Date.today
+      (now .. now + 14).each do |date|
+        url = day_url(chan_id, date)
+#        puts url
+  #      url = '/home/han/program2007'
+        page = fetch(url)
+        avond = false
+        page.search('div#program_complete//tr').each do |prog|
+  #        puts prog
+
+          times = prog.at('span.tv-sendung-uhrzeit')
+          next unless times
+  #        puts times.inner_text
+          titel = prog.at("td/a.tv-sendung-titel")
+          id = titel['href'][/detailPopup\('(\d+)'/, 1]
+          href = "/program2007?format=detail&sid=#{id}"
+#          puts "#{id}: #{@channelhash[id]}"
+          found += 1
+          next if @channelhash[id]
+          begin
+            @channelhash[id] = program = grab_detail(href)
+            fetched += 1
+          rescue
+            STDERR.puts href, page, '====='
+            raise
+          end
+          times = times.inner_text
+          start = times.to_i
+          avond = true if start > 17 
+          program['times'] = times
+          program['title'] = titel.inner_text.to_utf
+          program['datum'] = date.to_s
+          program['evening'] = avond
+
+  #        exit
+        end
+      end
+      save_object(@channelhash, cachefile(chan_id)) if fetched > 0
+      found
+    end
+    # transform returns an array of special hashes (obtained with proghash)
+    # 
+    def transform(chan_id)
+      progdata_array = Array.new
+      @channelhash.each_pair do |id, entry|
+        begin
+          progdata = proghash(entry, chan_id)
+#          pp progdata
+#          pp entry
+          start, stop = entry['times'].split('-')
+          date = Date.parse(entry['datum'])
+          startuur, startmin, stopuur, stopmin = entry['times'].split(/[-.]/).map{|x| x.to_i}
+          if startuur < 7 && entry['evening']
+            date += 1
+          end
+          progdata['start'] = Time.local(date.year, date.month, date.day, startuur, startmin)
+          progdata['stop']  = Time.local(date.year, date.month, date.day, stopuur, stopmin)
+          if progdata['stop'] < progdata['start']
+            progdata['stop'] += Dag
+          end
+          date_stats(chan_id, progdata['start'])
+#          puts progdata['stop'], progdata['start']
+          if entry['desc'].index("\t\t\t\t>") || entry['desc'].empty? && ! entry['desc-s'].empty?             
+            progdata['desc'] = entry['desc-s']
+          end
+          if entry['info']
+#            puts entry['info']
+            details = entry['info'].split(';').map{|x| x.strip}
+            role = nil
+            details.each do |det|
+              year = nil
+              case det[0,2]
+                when 'R:'
+                  role = 'director'
+                  det = det[2..-1].strip
+                when 'D:'
+                  role = 'actor'
+                  det = det[2..-1]
+                when 'O:'
+                  year = det[-5..-1].to_i
+                else
+                  year = det[-5..-1].to_i
+              end
+              if year && year > 1900
+                progdata['date'] = year.to_s
+              elsif role
+                (progdata['credits'][role] ||= Array.new) << det.gsub(/\(.*\)/,'').strip
+              end
+            end
+            progdata['audio']['stereo'] = 'stereo' if entry['Stereo']
+            progdata['subtitles']['type'] = 'teletext' if entry['Untertitel']
+            progdata['video']['aspect'] = '16:9' if entry['Breitbild']
+          end
+          
+          progdata_array << progdata
+        rescue StandardError => exc
+          STDERR.puts exc, exc.message, exc.backtrace
+          PP.pp(entry, STDERR)
+          raise
+        end
+      end
+      progdata_array
+    end
+  end
+
+end
+XMLTV::TvtodayGrabber.new.run