xmltv 0.8.1

data/lib/xmltv/xmltv.rb

@@ -0,0 +1,737 @@
+#!/usr/bin/ruby -w
+
+require 'rubygems'
+require 'hpricot'
+require 'rexml/document'
+require 'yaml'
+require 'date'
+require 'fileutils'
+require 'timeout'
+require 'open-uri'
+require 'pp'
+require 'iconv'
+require 'set'
+require 'optparse'
+require 'optparse/time'
+require 'ostruct'
+
+class Time
+  public :to_date
+end
+module MailSender
+  def linuxmail(to, subj, body)
+    from = `/usr/bin/id --name --user`.chomp
+    header = ["To: #{to}", "From: #{from}", "Subject: #{subj}"].join("\n")
+    mess = [header,"", body].join("\n")
+    puts mess
+    IO.popen("/usr/sbin/sendmail -t", 'w') do |h|
+      h.puts mess
+    end
+  end
+  def sendmail(*args)
+    if test(?x, '/usr/sbin/sendmail')
+      linuxmail(*args)
+    else
+      raise "FIXME: Don't know how to send mail"
+    end
+  end
+  module_function :sendmail, :linuxmail
+end
+class String
+  UTF = 'utf-8'
+  ISO = 'iso-8859-15'
+  TO_UTF = Iconv.new(UTF, ISO)
+  CK_UTF = Iconv.new(UTF, UTF)
+  def to_utf
+    TO_UTF.iconv(self)
+  end
+  def ck_utf
+    CK_UTF.iconv(self)
+  end
+end
+
+class Date
+  Maanden = [nil] + %w{ januari februari maart april mei juni juli augustus september oktober november december }
+  Dagen = %w{ zondag maandag dinsdag woensdag donderdag vrijdag zaterdag }
+  def self.dutch(instring)
+    nu = today
+    defjaar, defmaand, defdag, weekdag = nu.year, nu.month, nu.day, nu.wday
+    jaar = maand = dag = nil
+    string = instring.gsub(/[[:cntrl:][:punct:]]/, '').downcase
+    string.split.each do |elem|
+      case elem
+        when /[[:alpha:]]+/
+          if Maanden.include?(elem)
+            maand = Maanden.index(elem)
+            if maand < nu.month
+              defjaar += 1
+            end
+          elsif Dagen.include?(elem)
+            wd = Dagen.index(elem) - weekdag
+            wd += 7 if wd < 0
+            nu += wd
+            defjaar, defmaand, defdag, weekdag = nu.year, nu.month, nu.day, nu.wday
+          end
+        when /[[:digit:]]+/
+          ent = elem.to_i
+          if ent <= 31
+            dag = ent
+          elsif ent > 1900
+            jaar = ent
+          end
+      end
+    end
+    new(jaar || defjaar, maand || defmaand,dag || defdag)
+  end
+end
+        
+
+module XMLTV
+  class ValidateError < Exception; end
+  class BadChannelError < Exception; end
+  class BadSiteError < Exception; end
+  class Progdata < Hash
+    def self.new
+      super {|h,v| h[v] = Hash.new }
+    end
+  end
+  VERSION = '0.8.1'
+  Progdir = ($LOAD_PATH.find {|x| test(?f, "#{x}/xmltv/xmltv.rb")} || 'lib') + '/xmltv/sites'
+  Sites = Dir["#{Progdir}/*.rb"].map{|x| x[0..-4]}.map{|x| File.basename(x)}
+  class XmltvOptparser
+
+    #
+    # Return a structure describing the options.
+    #
+    def self.parse(args)
+      options = OpenStruct.new
+      options.basedir = "#{ENV['HOME']}/.xmltv"
+      options.validate = true
+
+      opts = OptionParser.new do |opts|
+        opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+        opts.separator ""
+        opts.separator "Specific options:"
+
+        opts.on("-a", "--list-all",
+                "List all available channels") do |av|
+          options.available = av
+          options.action = true
+        end
+
+        opts.on("--add x,y,z", Array, "Add channels to config") do |list|
+          options.add = list
+          options.action = true
+        end
+        opts.on("--delete x,y,z", Array, "Delete channels from config") do |list|
+          options.del = list
+          options.action = true
+        end
+        opts.on("-l", "--list", "List configured channels") do |list|
+          options.list = list
+          options.action = true
+        end
+        opts.on("-c", "--config-file FILENAME", "Configuration file") do |list|
+          options.config = list
+        end
+        opts.on( "--basedir DIRNAME", "Basedir (#{options.basedir})") do |list|
+          options.basedir = list
+        end
+        opts.on( "--no-validate", "Do not validate") do |v|
+          options.validate = false
+        end
+        opts.on( "--mailto USER", "Mail exception") do |user|
+          options.mailto = user
+        end
+
+        opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+          options.verbose = v
+        end
+
+        opts.separator ""
+        opts.separator "Common options:"
+
+        opts.on_tail("-h", "--help", "Show this message") do
+          STDERR.puts opts
+          exit
+        end
+
+        opts.on_tail("--version", "Show version") do
+          options.version = true
+        end
+      end
+
+      opts.parse!(args)
+      options
+    end
+  end
+  XmltvOptions = XmltvOptparser.parse(ARGV)
+  
+  
+  class XmlWriter
+    With_lang = [  'title','sub-title','desc','category','language','orig-language','country','premiere','last-chance']
+    Programme_dtd = [
+      ['title' ],
+      [ 'sub-title' ],
+      [ 'desc' ],
+      [ 'credits', nil, [
+        'director', 
+        'actor',
+        'writer',
+        'adapter',
+        'producer',
+        'presenter', 
+        'commentator',
+        'guest'
+        ]
+      ],
+      ['date'],
+      ['category' ] ,
+      ['language' ] ,
+      ['orig-language' ] ,
+      [ 'length', [ 'units' ] ],
+      [ 'icon', [ 'src', 'width', 'height' ] ], 
+      [ 'url'],
+      [ 'country' ] , 
+      [ 'episode-num', [ 'system' ] ], 
+      [ 'video', [], [
+        'present',
+        'colour', 
+        'aspect',
+        'quality'
+        ]
+      ],  
+      [ 'audio' , [],  [
+        'present',
+        'stereo'
+        ]
+      ],
+      ['previously-shown', ['start', 'channel' ]],
+      ['premiere' ],
+      [ 'last-chance' ],
+      [ 'new' ],
+      [ 'subtitles', [ 'type' ], [ 'language' ] ], 
+      [ 'rating', [ 'system' ] ,  [ 'value', 'icon' ] ],
+      [ 'star-rating', [],  [ 'value', 'icon' ] ]
+    ]
+    def initialize(grabber)
+      @grabber = grabber
+      @doc = doc = REXML::Document.new
+      doc << REXML::XMLDecl.new("1.0", "UTF-8")
+      dtd = grabber.config['dtd'] || 'file:///usr/share/xmltv/xmltv.dtd'
+      doc << REXML::DocType.new('tv', %Q{SYSTEM "#{dtd}"})
+      @el_tv = doc.add_element( 'tv' )
+      @el_tv.attributes['generator-info-name'] = 'xmltv.rb'
+    end
+    def add_dtd(progdata)
+      raise ArgumentError.new("add_dtd must have a Progdata") unless progdata.is_a?(Progdata)
+      p = @el_tv.add_element('programme')
+      %w{ start stop }.each do |word|
+        p.attributes[word] = progdata[word].strftime( '%Y%m%d%H%M%S %Z')
+      end
+      p.attributes['channel'] = progdata['channel']
+      Programme_dtd.each do |el|
+        name , attrs , elements = el
+#        STDERR.puts name
+        next if (cur = progdata[name]).empty?
+        el = p.add_element(name)
+        el.text = cur if cur.is_a? String
+        if With_lang.include? name
+          el.attributes['lang'] = @grabber.lang
+        end
+        if attrs
+          attrs.each do |a|
+            el.attributes[a] = cur[a] if cur[a]
+          end
+        end
+        if elements
+          elements.each do |e|
+            case cur[e]
+              when String
+                el.add_element(e).text = cur[e]
+              when Array
+                cur[e].each do |r|
+                  el.add_element(e).text = r
+                end
+            end
+          end
+        end
+      end
+    end
+    def write_file(pda, chan_id)
+      pda.each do |progdata|
+        begin
+          add_dtd(progdata)
+        rescue StandardError => exc
+          STDERR.puts exc, exc.message, exc.backtrace, '===='
+          PP.pp(progdata, STDERR)
+          raise
+        end
+      end
+      file = ">#{@grabber.outputfile(chan_id)}"
+      errorfile = '/var/tmp/xmllint-errors'
+      if XmltvOptions.validate
+        IO.popen(" ( xmllint --valid - | tv_sort #{file} ) 2>#{errorfile}", 'w') do |h|  
+          write_xml h
+        end
+        if test(?s, errorfile)
+          raise ValidateError.new("zie #{errorfile}")
+        end
+      else
+        File.open("#{@grabber.outputfile(chan_id)}", 'w') do |h|
+          write_xml h
+        end
+      end
+    end
+    def write_xml(h, indent = -1)
+      @doc.write(h, indent)
+      h.puts
+    end
+
+  end
+  class Grabber
+    Dag = 24 * 60 * 60
+    Vandaag = Date.today
+    attr_accessor :myname, :chnbasedir, :spooldir, :channel_list, :lang, :generator, :base_url, :config_file_name
+    attr_accessor :config, :all_channels, :reject_file_name
+    mtv = 
+      begin
+        YAML.load_file("#{XmltvOptions.basedir}/mythtv_chns.yaml")
+      rescue Errno::ENOENT
+        Hash.new
+      end
+    MythTV = Hash.new
+    mtv.each do |myth, info|
+      info[0].each do |file|
+        MythTV[file] = myth
+      end
+    end
+    
+    def grab_channel(chan_id)
+      STDERR.puts "Grabber must implement grab_channel(chan_id)"
+      exit
+    end
+    def transform(chan_id)
+      STDERR.puts "Grabber must implement transform(chan_id)"
+      exit
+    end
+
+    def initialize
+      @hits = 0
+#      @myname = File.basename($PROGRAM_NAME).gsub(/\..*/, '')
+      @myname = self.class.to_s[7..-8].downcase
+      @chnbasedir = "#{XmltvOptions.basedir}/#{myname}"
+      @config_file_name = XmltvOptions.config || "#{chnbasedir}/config"
+      @reject_file_name = "#{chnbasedir}/rejects"
+      @config = load_config_file
+      @lang = 'nl'
+      @generator = "#{myname}.#{lang}"
+      @base_url = "http://www.#{generator}"
+      @spooldir = config['spooldir'] || "#{chnbasedir}/spool"
+      [ chnbasedir, spooldir ].each do |dir|
+        FileUtils.mkdir_p(dir) unless test(?d, dir)
+      end
+      @channel_list = "#{chnbasedir}/channel_list"
+      @all_channels = load_channel_file(channel_list)
+    end
+    def version
+      "XMLTV::Grabber version 0.8"
+    end
+    
+    def outputfile(chan_id)
+      "#{spooldir}/#{channel_name(chan_id)}.xml"
+    end
+    def channel_name(chan_id)
+      "#{chan_id}.#{myname}.#{lang}"
+    end
+    def channel_display(chan_id)
+      all_channels[chan_id]
+    end
+    def cachefile(chan_id)
+      "#{chnbasedir}/#{channel_name(chan_id)}.yaml"
+    end
+    def load_cachefile(chan_id)
+      begin
+        YAML.load_file(cachefile(chan_id))
+      rescue Errno::ENOENT
+        Hash.new
+      end
+    end
+    def check_argv(args)
+      case args[0]
+        when 'all'
+          all_channels.keys
+        when 'new'
+          XmltvOptions.only_new = true
+          all_channels.keys
+        else
+          args
+      end
+    end
+    
+    def get_channels
+      clean_cache_dir
+      if ! ARGV.empty?
+        ARGV.replace(check_argv(ARGV))
+      else
+        config_channels
+      end
+    end
+
+    def load_config_file
+      begin
+        YAML.load_file(config_file_name)
+      rescue
+        h = Hash.new
+        h['channels'] = []
+        h
+      end
+    end
+
+    def load_channel_file(fn)
+      if (fn)
+        begin
+          YAML.load_file(fn)
+        rescue  Errno::ENOENT
+          fetch_all_channels
+        end
+      end
+    end
+    def save_config
+      loop do
+        if test(?f, config_file_name)
+          File.open(config_file_name, 'w') { |h| h.puts config.to_yaml }
+          return
+        end
+        dir = File.dirname(config_file_name)
+        if test(?d, dir)
+          FileUtils.touch config_file_name
+        elsif test(?e, dir)
+          raise "#{dir} exists, but isn't a directory"
+        else
+          FileUtils.mkdir_p dir
+        end
+      end
+    end
+      
+    
+    def config_channels
+      rsl = config['channels']
+      if rsl.empty?
+        all_channels.keys
+      else
+        rsl
+      end
+    end
+    def clean_spool_dir
+      xmlfiles = config_channels.map{|x| outputfile(x) }
+      Dir["#{spooldir}/*.xml"].each do |fn|
+        unless xmlfiles.include? fn
+          STDERR.puts("removed #{fn} -- not configured")
+          File.unlink fn
+        end
+      end
+    end
+      
+    
+    def clean_cache_dir
+      clean_spool_dir
+      last_time = config['cleaned']
+      if last_time.is_a?(Date) && last_time >= Vandaag
+        return
+      end
+      count = 0
+      channels = config_channels.map{|x| cachefile(x) }
+      Dir["#{chnbasedir}/*.yaml"].each do |fn|
+        unless channels.include? fn
+          STDERR.puts("removed #{fn} -- not configured")
+          File.unlink fn
+          next
+        end
+        cache = YAML.load_file(fn)
+        dels = clean_cache(cache)
+        File.open(fn, 'w') {|h| h.puts cache.to_yaml} if dels > 0
+        count += dels
+        config['cleaned'] = Vandaag
+      end
+      STDERR.puts("#{myname}: Removed #{count} entries") if count > 0
+
+    end
+    
+
+
+
+    def proghash(entry, chan_id)
+      progdata = Progdata.new
+      entry.keys.each do |k|
+        dtd = XmlWriter::Programme_dtd.assoc(k)
+        if dtd && dtd.size == 1
+          progdata[k] = entry[k]
+        end
+      end
+      cnm = channel_name(chan_id)
+      progdata['channel'] = MythTV[cnm] || cnm
+      progdata
+    end
+    
+    def reject(*args)
+      File.open(reject_file_name, 'a') do |h|
+        h.puts Time.now
+        args.each do |arg|
+          case arg
+            when String, Time, Integer
+              h.puts arg
+            else
+              PP.pp(arg, h)
+          end
+        end
+        h.puts '===='
+      end
+    end
+        
+              
+
+
+    def fix_times(pda)
+      errors = 0
+      startwith = pda.size
+      pda.delete_if do |x| 
+        rsl = x.has_key?('stop') && x['stop'] <= x['start']
+        reject('invalid', x) if rsl
+        rsl
+      end
+      starttimes = pda.sort do |x, y| 
+        if (a = x['start']) != (b = y['start'])
+          a <=> b
+        else
+          a = x.has_key?('stop') ? x['stop'] : x['start']
+          b = y.has_key?('stop') ? y['stop'] : y['start']
+          b <=> a
+        end
+      end
+      starttimes.each_with_index do |entry, idx|
+  #      dump(entry, '++++++++++++++') if entry.has_key?('title') && entry['title'] == 'NOS Studio Voetbal'
+        nxt = starttimes[idx + 1]
+        next unless nxt
+        unless entry.has_key?('stop') && entry['stop'] <= nxt['start']
+          errors += 1
+          entry['stop'] = nxt['start'].dup
+        end
+      end
+      unless starttimes[-1].has_key?('stop')
+        starttimes[-1]['start'] = nil
+      end
+      pda.delete_if do |x| 
+        rsl = x['start'].nil? ||  x['stop'] <= x['start']
+        reject('removed', x) if rsl
+        rsl
+      end
+      @rejects = startwith - pda.size
+      errors
+    end
+    def fetch(url)
+      @hits += 1
+      tries = 0
+      begin
+        open(url) { |h| Hpricot(h) }
+      rescue Errno::ECONNREFUSED, ::Timeout::Error
+        tries += 1
+        sleep 2
+        retry if tries <= 3
+        raise BadSiteError.new
+      end
+    end
+    def save_object(obj, filename)
+      File.open(filename, 'w') do |h|
+        h.puts obj.to_yaml
+      end
+    end
+    def save(*args)
+      file = "/tmp/xmltv_exception.#{$$}"
+      args.each_with_index do |el, i|
+        File.open("#{file}-#{i}", 'w') do |h|
+          h.puts el
+        end
+      end
+      STDERR.puts "See #{file}-*"
+    end
+
+    def dump(*args)
+      args.each do |arg|
+        PP.pp(arg,STDERR)
+      end
+    end
+    def date_stats(chan, time)
+      @days_av[chan].add(time.to_date)
+    end
+    def check_channel(chan_id)
+      all_channels.has_key?(chan_id)
+    end
+    
+    def add_channels_to_config
+      XmltvOptions.add.each do |item|
+        if (f = all_channels.has_key?(item)) && ! config['channels'].include?(item)
+          config['channels'] << item
+          STDERR.puts("added #{item} #{channel_display(item)}") if XmltvOptions.verbose
+        else
+          reason = f ? "already included" : "not in channel list"
+          STDERR.puts "#{item} #{reason}" 
+        end
+      end
+      save_config
+    end
+    
+    def delete_channels_from_config
+      XmltvOptions.del.each do |item|
+        if config['channels'].include?(item)
+          config['channels'].delete(item)
+          STDERR.puts("deleted #{item} #{channel_display(item)}") if XmltvOptions.verbose
+        else
+          STDERR.puts "#{item} not in config list" 
+        end
+      end
+      save_config
+    end
+    def printline(it)
+      printf("%-5s %s\n", it, channel_display(it))
+    end
+    def do_list(array)
+      array.sort_by{|x| channel_display(x)}.each do |item|
+        printline item
+      end
+    end
+    def list_config
+      if config['channels'].empty?
+        explan = channel_list ? 'yet' : 'needed'
+        puts "No configuration #{explan} for #{myname}"
+      else
+        puts "Configured channels for #{myname}"
+        do_list(config['channels'])
+      end
+    end
+    def list_all
+      puts "Available channels for #{myname}"
+      do_list(all_channels.keys)
+    end
+    def do_options
+      if XmltvOptions.list
+        list_config
+      elsif XmltvOptions.available   
+        list_all
+      else
+        if XmltvOptions.add
+          add_channels_to_config
+        end
+        if XmltvOptions.del
+          delete_channels_from_config
+        end
+        list_config
+      end
+      exit
+    end
+
+    def report(channel, nprogs, errors)
+      STDERR.printf("  %s %-6.6s %-20.20s %5d %5d %5d %5d %8d %5d\n",
+        Time.now.strftime("%H:%M:%S"),
+        channel,
+        channel_display(channel),
+        nprogs,
+        @hits,
+        @days_av[channel].size,
+        errors,
+        test(?s, outputfile(channel)),
+        @rejects)
+        
+      @hits = 0
+    end
+        
+    def run
+      if XmltvOptions.version
+        puts version
+        exit
+      end
+      do_options if XmltvOptions.action
+      if channel_list && ! test(?f, config_file_name)   # We don't want to grab _all_ channels, surely ?
+        list_all
+        puts "\n Do #{$PROGRAM_NAME} --add chn1,chn2,chn3... to create a configuration file"
+        exit
+      end
+      ARGV.delete_if do |arg|
+        begin
+          send(arg)
+          true
+        rescue NoMethodError
+          false
+        end
+      end
+      trap('INT') do
+        STDERR.puts "\nInterrupted\n"
+        raise ArgumentError.new('interrupt')
+#        exit!
+      end
+      
+      begin
+        channels = get_channels
+        STDERR.printf("\n%-40s prgs   hts  days  errs   bytes    rej\n", "#{Date.today}: #{myname}")
+        channels.each do |channel|
+          next if XmltvOptions.only_new && test(?f, outputfile(channel))
+          @days_av = Hash.new { |h,k| h[k] = Set.new }
+          @rejects = 0
+          writer = XmlWriter.new(self)
+          unless check_channel(channel)
+            STDERR.puts "No such channel #{channel}"
+            next
+          end
+          begin  
+            nprogs = grab_channel(channel)
+            pda = transform(channel)
+            errors = fix_times(pda)
+          rescue BadChannelError
+            STDERR.puts "Zie #{reject_file_name}"
+            next
+          end
+          writer.write_file(pda, channel)
+          report(channel, nprogs, errors)
+        end
+      rescue SystemExit
+        
+      rescue ::Exception => exc
+        STDERR.puts exc.class, exc.message, exc.backtrace
+        if XmltvOptions.mailto
+          MailSender.sendmail(XmltvOptions.mailto, "Xmltv exception", [exc.class, exc.message, exc.backtrace].join("\n"))
+        end
+        raise
+      ensure
+      end
+      save_config
+    end
+
+  end
+  def go
+    if XmltvOptions.add || XmltvOptions.del || XmltvOptions.config
+      STDERR.puts "You must specify a site to configure channels"
+      exit
+    end
+    configs = Dir["#{XmltvOptions.basedir}/*"].select{|x| File::Stat.new(x).directory? }.map{|x| File.basename(x)}
+    if configs.empty?
+      STDERR.puts "No configs found in #{XmltvOptions.basedir}"
+    end
+    todo = configs & Sites
+    exit if todo.empty?
+    todo.sort.each do |grabber|
+  #    puts grabber
+      begin
+        require "xmltv/sites/#{grabber}"
+      rescue SystemExit => exc
+        raise unless exc.success?
+      end
+    end
+  end
+  
+  module_function :go 
+end
+if __FILE__ == $PROGRAM_NAME
+  XMLTV::go
+end
+