apple-tv-converter 0.4.4 → 0.5.1

data/lib/apple_tv_converter/subtitles_fetcher/opensubtitles.rb

@@ -20,75 +20,67 @@ module AppleTvConverter
       end
 
       def logout
-        begin
-          response = @server.call("LogOut", get_token)
-          parse_response! response
-          @token = nil if response[:success]
-        rescue EOFError
-          logout
-        rescue XMLRPC::FaultException => e
-          puts "Error:"
-          puts e.faultCode
-          puts e.faultString
-          raise e
-        end
+        response = make_call("LogOut", get_token)
+        parse_response! response
+        @token = nil if response[:success]
       end
 
       def search_subtitles(media, &block)
-        options = [
+        language_options = languages.map(&:to_s).join(',') if languages.any?
+        options = []
+
+        # Query by movie hash
+        options << {
           :moviehash => media.movie_hash.to_s,
           :moviebytesize => media.movie_file_size.to_s
-        ]
-        language_options = languages.map(&:to_s).join(',') if languages.any?
+        }
+        # Query by movie name
+        options << { :query => media.show }
+        # and IMDB id if present
+        options.last[:imdb_id] = media.imdb_id if media.imdb_id
+
+        # Add common options
+        options.each do |query_option|
+          query_option[:sublanguageid] = language_options if language_options
+          query_option[:season] = media.season if media.is_tv_show_episode?
+          query_option[:episode] = media.number if media.is_tv_show_episode?
+        end
 
-        options.first[:sublanguageid] = language_options if language_options
         response = search_for_subtitles(media, options)
-
-        if response[:success]
-          if response['data']
-            Opensubtitles.subtitles[media] = response['data']
-            block.call response['data'] if block
-          else
-            # Could not find matches by hash, try by name (and season/episode)
-            options = [ :query => media.show ]
-
-            options.first[:season] = media.season if media.is_tv_show_episode?
-            options.first[:episode] = media.number if media.is_tv_show_episode?
-            options.first[:sublanguageid] = language_options if language_options
-
-            response = search_for_subtitles(media, options)
-
-            if response[:success] && response['data']
-              Opensubtitles.subtitles[media] = response['data']
-              block.call response['data'] if block
-            end
-          end
+        if response[:success] && response['data']
+          Opensubtitles.subtitles[media] = response['data']
+          block.call response['data'] if block
         end
       end
 
+      def has_found_subtitles?(media)
+        (Opensubtitles.subtitles[media] && Opensubtitles.subtitles[media].any?) == true
+      end
+
       def download_subtitles(media, &block)
+        return unless has_found_subtitles? media
+
         data = Opensubtitles.subtitles[media]
         media_subtitles = filter_subtitles(data, media)
+
+        # If we have subtitles matched by moviehash, get only the first and ignore the rest
+        # otherwise, get all
+        media_subtitles = Hash[*media_subtitles.map { |language, subs| [language, [ subs.detect { |s| s['MatchedBy'] } || subs ].flatten ] }.flatten(1) ]
+
         block.call :search, media_subtitles if block
 
-        # We now have only one subtitle per language code, so start downloading
-        media_subtitles.each do |language_code, subtitle|
-          block.call :downloading, subtitle
-          download_subtitle(media, subtitle)
-          block.call :downloaded, subtitle
+        # We now have one or many subtitles per language code, so start downloading
+        media_subtitles.each do |language_code, subtitles|
+          subtitles.each do |subtitle|
+            block.call :downloading, subtitle
+            download_subtitle(media, subtitle)
+            block.call :downloaded, subtitle
+          end
         end
       end
 
       def status
-        begin
-          @server.call('ServerInfo')
-        rescue EOFError
-          status
-        rescue XMLRPC::FaultException => e
-          puts "Error:"
-          puts e.faultCode
-          puts e.faultString
-        end
+        make_call('ServerInfo')
       end
 
       private
@@ -100,19 +92,10 @@ module AppleTvConverter
         def logged_in? ; return !@token.nil? ; end
 
         def login
-          begin
-            response = @server.call("LogIn", '', '', '', USER_AGENT)
-            parse_response! response
+          response = make_call("LogIn", '', '', '', USER_AGENT)
+          parse_response! response
 
-            @token = response['token'] if response[:success]
-          rescue EOFError
-            login
-          rescue XMLRPC::FaultException => e
-            puts "Error:"
-            puts e.faultCode
-            puts e.faultString
-            raise e
-          end
+          @token = response['token'] if response[:success]
         end
 
         def get_token
@@ -120,80 +103,131 @@ module AppleTvConverter
           return @token
         end
 
-        def filter_subtitles(data, media)
-          media_subtitles = data.select { |s| s['SubFormat'].downcase == 'srt' } # Filter by format
-          media_subtitles = media_subtitles.select { |s| languages.empty? || languages.include?(s['SubLanguageID']) } # Filter by language
+        def normalize(string) ; return string.gsub(/[^0-9a-z ]/i, '').gsub(/\s+/, ' ').downcase.strip ; end
 
-          exact_match = media_subtitles.select do |s|
-            !File.basename(media.original_filename).downcase.index(s['MovieReleaseName'].downcase).nil? ||
-            !File.basename(media.original_filename).downcase.index(s['SubFileName'].gsub(/\..*?$/, '').downcase).nil?
+        def filter_subtitles(data, media)
+          # "MatchedBy" -> "moviehash"
+          # "MatchedBy" -> "imdbid"
+          # "MatchedBy" -> "fulltext"
+
+          # Define priorities by match type
+          data.each do |s|
+            s[:priority] = case s['MatchedBy']
+              when 'moviehash'  then 100
+              when 'imdbid'     then 200
+              when 'fulltext'   then 300
+              else                   400
+            end
           end
 
-          # We found exact matches on the movie name, so ignore the rest
-          media_subtitles = exact_match if exact_match.any?
+
+          # Order the subtitles first by lowest priority (my match)
+          # and then by download count (descending). This way, we'll get the
+          # best, top downloaded match on top
+          media_subtitles = data.sort { |c, d| [d[:priority], c['SubDownloadsCnt'].to_i] <=> [c[:priority], d['SubDownloadsCnt'].to_i] }.reverse
+
+
+          # Get only unique subtitle entries (we can have more than one)
+          # due to different 'MatchedBy'
+          media_subtitles = media_subtitles.uniq { |s| s['IDSubtitle'] }
+
+          # Filter by subtitles format (srt)
+          media_subtitles = media_subtitles.select { |s| s['SubFormat'].downcase == 'srt' }
+          # Filter by number of discs (1)
+          media_subtitles = media_subtitles.select { |s| s['SubSumCD'] == '1' }
+          # Filter by language
+          media_subtitles = media_subtitles.select { |s| languages.empty? || languages.include?(s['SubLanguageID']) }
+          # Filter by movie name (unless it's an episode, as the movie name can be the episode's title)
+          media_subtitles = media_subtitles.select { |s| s['MatchedBy'] == 'moviehash' || normalize(s['MovieName']) == normalize(media.show) } unless media.is_tv_show_episode?
+
+          # exact_match = media_subtitles.select do |s|
+          #   !File.basename(media.original_filename).downcase.index(s['MovieReleaseName'].downcase).nil? ||
+          #   !File.basename(media.original_filename).downcase.index(s['SubFileName'].gsub(/\..*?$/, '').downcase).nil?
+          # end
+
+          # # We found exact matches on the movie name, so ignore the rest
+          # media_subtitles = exact_match if exact_match.any?
+
           # Group the subtitles by language code
           media_subtitles = media_subtitles.group_by { |a| a['SubLanguageID'] }
 
-          # Since we can have more than one subtitle per language that matches our movie
-          # order the grouped subtitles by download count (descending), and keep only
-          # the first (we basically going with the majority of the people)
-          return Hash[*(media_subtitles.map {|a,b| [a, b.sort { |c, d| c['SubDownloadsCnt'].to_i <=> d['SubDownloadsCnt'].to_i }.reverse.first] }).flatten]
+          all_subtitles = Hash[*media_subtitles.flatten(1)]
+
+          return all_subtitles
         end
 
         def search_for_subtitles(media, options)
-          begin
-            response = @server.call("SearchSubtitles", get_token, options)
-            parse_response! response
+          response = make_call("SearchSubtitles", get_token, options)
+          parse_response! response
 
-            return response
-          rescue EOFError
-            logout
-          rescue XMLRPC::FaultException => e
-            puts "Error:"
-            puts e.faultCode
-            puts e.faultString
-            raise e
-          end
+          return response
         end
 
         def download_subtitle(media, subtitle)
-          begin
-            response = @server.call("DownloadSubtitles", get_token, [ subtitle['IDSubtitleFile'] ])
-            parse_response! response
-
-            if response[:success]
-              data = response['data']
-
-              if data
-                data.each do |subtitle_data|
-                  # Decode Base64 encoded gzipped data
-                  zip_data = Base64.decode64(subtitle_data['data'])
-                  # UnGZip it
-                  unzipped_data = Zlib::GzipReader.new(StringIO.new(zip_data)).read
-                  # Write it to a new file
-                  File.open(media.get_new_subtitle_filename(subtitle['SubLanguageID'], subtitle_data['idsubtitlefile']), 'wb') { |file| file.write(unzipped_data) }
-                end
+          response = make_call("DownloadSubtitles", get_token, [ subtitle['IDSubtitleFile'] ])
+          parse_response! response
+
+          if response[:success]
+            data = response['data']
+
+            if data
+              data.each do |subtitle_data|
+                # Decode Base64 encoded gzipped data
+                zip_data = Base64.decode64(subtitle_data['data'])
+                # UnGZip it
+                unzipped_data = Zlib::GzipReader.new(StringIO.new(zip_data)).read
+                # Write it to a new file
+                File.open(media.get_new_subtitle_filename(subtitle['SubLanguageID'], subtitle_data['idsubtitlefile']), 'wb') { |file| file.write(unzipped_data) }
               end
             end
-          rescue EOFError
-            download_subtitle subtitle
+          end
+        end
+
+        def download(url)
+          Net::HTTP.get(URI.parse(url))
+        end
+
+        def make_call(function, *parameters)
+          do_make_call function, 0, parameters
+        end
+
+        def do_make_call(function, retries, *parameters)
+          begin
+            # Flatten the parameters to the correct depth
+            @server.call(*[function, parameters.flatten(1)].flatten(1))
+          rescue EOFError => e
+            if retries < 3
+              # retry
+              puts "Error (EOFError): retrying"
+              do_make_call function, retries + 1, *parameters
+            else
+              puts "Error (EOFError): retried 3 times, giving up"
+              raise e
+            end
           rescue XMLRPC::FaultException => e
-            puts "Error:"
+            puts "Error (XMLRPC::FaultException):"
             puts e.faultCode
             puts e.faultString
             raise e
+          rescue RuntimeError => e
+            if retries < 3
+              # retry
+              puts "Error (RuntimeError, most likely 503): retrying after 2 seconds"
+              sleep 2
+              do_make_call function, retries + 1, *parameters
+            else
+              puts "Error (RuntimeError, most likely 503): retried 3 times, giving up"
+
+              raise e
+            end
           rescue Exception => e
-            puts "Error:"
-            # ap e
-            # ap e.message
+            puts "Error (#{e.class})"
+            puts e.message
 
             raise e
           end
         end
 
-        def download(url)
-          Net::HTTP.get(URI.parse(url))
-        end
 
         def parse_response!(response)
           # Clear the token in case of some errors

data/lib/apple_tv_converter/tv_db_fetcher.rb

@@ -0,0 +1,261 @@
+module AppleTvConverter
+  class TvDbFetcher
+    require 'httparty'
+    require 'yaml'
+    require 'net/http'
+    require 'zip/zip'
+    require 'xml'
+
+    include HTTParty
+
+    base_uri 'thetvdb.com/api'
+
+    def self.search(media, interactive = true, language = 'en')
+      get_updates_from_server
+
+      if media.tvdb_id
+        show_id = media.tvdb_id
+      else
+        data = load_config_file('show_ids') || {}
+
+        # http://thetvdb.com/api/GetSeries.php?seriesname=
+        unless data.has_key?(media.show)
+          show_ids = get_and_parse_data_from_server('show_ids', '/GetSeries.php', { :query => { :seriesname => media.show } }, ['Data', 'Series']) do |loaded_data|
+            loaded_data = [loaded_data].flatten
+
+            data[media.show] = if loaded_data.length > 1 && interactive
+              choice = 0
+              puts "\n  *"
+
+              while true
+                puts %Q[  | Several shows found, choose the intended one:]
+
+                loaded_data.each_with_index do |item, index|
+                  puts "  | #{(index + 1).to_s.rjust(loaded_data.length.to_s.length)} - #{item['SeriesName']} (id: #{item['seriesid']})"
+                  puts "  | #{' '.rjust(loaded_data.length.to_s.length)}   AKA: #{item['AliasNames']}" if item['AliasNames']
+                end
+
+                printf "  |\n  *- What's your choice (1..#{loaded_data.length})? "
+                choice = STDIN.gets.chomp.to_i
+
+                break if choice.between?(1, loaded_data.length)
+
+                puts "  | Invalid choice!"
+                puts "  |"
+              end
+
+              loaded_data[choice - 1]['seriesid']
+            else
+              loaded_data.first['seriesid']
+            end
+
+            # Return the new list sorted by show name
+            Hash[data.sort]
+          end
+        end
+
+        show_id = data[media.show]
+      end
+
+      if show_id.to_i > 0
+        # <mirrorpath_zip>/api/<apikey>/series/<seriesid>/all/<language>.zip
+        show_data = get_data(show_id, "/#{api_key}/series/#{show_id}/all/#{language}.zip", { :zip => true }) do |data|
+          show_data = xml_document_to_hash(XML::Document.string(data[language.to_s].gsub(/>\s*</im, '><')))
+          banners = xml_document_to_hash(XML::Document.string(data['banners'].gsub(/>\s*</im, '><'))) rescue { 'Banner' => [] }
+          actors = xml_document_to_hash(XML::Document.string(data['actors'].gsub(/>\s*</im, '><'))) rescue { 'Actor' => [] }
+
+          {
+            :series => show_data['Series'],
+            :episodes => [show_data['Episode']].flatten,
+            :banners => [banners['Banner']].flatten,
+            :actors => [actors['Actor']].flatten
+          }
+
+        end
+
+        return {
+          :episode => show_data[:episodes].detect { |ep| ep['SeasonNumber'].to_i == media.season.to_i && ep['EpisodeNumber'].to_i == media.number.to_i },
+          :show => show_data
+        }
+      end
+
+      return false
+    end
+
+    def self.get_poster(media)
+      local_file = File.join(AppleTvConverter.data_path, 'cache', 'tvdb', "#{media.tvdb_id}.jpg")
+
+      unless File.exists?(local_file)
+        artwork_filename = media.tvdb_movie[:show][:series]['poster'] || ''
+        artwork_filename = media.tvdb_movie_data('filename') || '' if artwork_filename.blank?
+        artwork_filename = "http://thetvdb.com/banners/#{artwork_filename}" if !artwork_filename.blank?
+
+        AppleTvConverter.copy artwork_filename, local_file unless artwork_filename.blank?
+      end
+
+      local_file
+    end
+
+    private
+
+      def self.api_key ; return '67FBF9F0670DBDF2' ; end
+      def self.local_cache_base_path
+        return File.expand_path(File.join(AppleTvConverter.data_path, 'cache', 'tvdb'))
+      end
+      def self.server_update_timestamp
+        @server_update_timestamp ||= load_config_file('update')
+
+        unless @server_update_timestamp
+          # http://thetvdb.com/api//Updates.php?type=none
+          @server_update_timestamp = get_data_from_server('/Updates.php', { :query => { :type => 'none' }})["Items"]["Time"] rescue nil
+          @server_update_timestamp = @server_update_timestamp.to_i unless @server_update_timestamp.nil?
+          save_config_file 'update', @server_update_timestamp
+        end
+
+        @server_update_timestamp
+      end
+
+      def self.load_config_file(filename)
+        full_filename = File.join(local_cache_base_path, filename =~ /\.yml$/ ? filename : "#{filename}.yml")
+        File.exists?(full_filename) ? YAML.load_file(full_filename) : nil
+      end
+
+      def self.save_config_file(filename, data)
+        full_filename = File.join(local_cache_base_path, filename =~ /\.yml$/ ? filename : "#{filename}.yml")
+        File.open(full_filename, 'w') { |f| f.write data.to_yaml }
+      end
+
+      def self.delete_config_file(filename)
+        full_filename = File.join(local_cache_base_path, filename =~ /\.yml$/ ? filename : "#{filename}.yml")
+        File.delete(full_filename) if File.exists?(full_filename)
+      end
+
+
+      def self.get_data_from_server(url, options = {})
+        AppleTvConverter.logger.debug "  -> Getting from server: #{url}"
+        cache = options.delete(:cache) || true
+        zip = options.delete(:zip) || false
+        response = self.get(url, options).parsed_response
+
+        if zip
+          filename = File.join(local_cache_base_path, 'zip_file.zip')
+
+          begin
+            File.open(filename, 'wb') { |f| f.write response }
+            response = {}
+
+            Zip::ZipFile.open(filename) do |zipfile|
+              zipfile.each do |entry|
+                unless entry.name.downcase["__macosx"]
+                  zip_data = zipfile.read(entry)
+                  response[entry.name.to_s.gsub(/\.xml$/i, '')] = zip_data
+                end
+              end
+            end
+          rescue => e
+            ap [e, e.backtrace]
+
+          ensure
+            FileUtils.rm_f filename if File.exists?(filename)
+          end
+        end
+
+        return response
+      end
+
+      def self.get_data(filename, url, url_options, response_indexes = [])
+        AppleTvConverter.logger.debug "-> Getting data: #{filename}"
+        data = load_config_file(filename)
+
+        unless data
+          data = get_data_from_server(url, url_options)
+
+          if data
+            begin
+              response_indexes.each { |idx| data = data[idx] }
+
+              data = yield(data) if block_given?
+
+              save_config_file filename, data
+            rescue
+              data = nil
+            end
+          end
+        else
+          # ap ['found on cache', filename, data]
+        end
+
+        return data
+      end
+
+      def self.get_and_parse_data_from_server(filename, url, url_options, response_indexes = [])
+        cache = url_options.delete(:cache) || true
+        data = get_data_from_server(url, url_options)
+
+        if data
+          begin
+            response_indexes.each { |idx| data = data[idx] }
+
+            data = yield(data) if block_given?
+
+            save_config_file filename, data if cache
+          rescue
+            data = nil
+          end
+        end
+      end
+
+      def self.get_updates_from_server(options = {})
+        get_and_parse_data_from_server('updates', '/Updates.php', { :query => { :type => 'all', :time => load_config_file('update')}, :cache => false }, ['Items']) do |data|
+          if data
+            # Delete each show's cached data
+            data['Series'].each do |show_id|
+              delete_config_file show_id
+              delete_config_file "#{show_id}.jpg"
+            end
+
+            # Save the new timestamp
+            save_config_file 'update', data['Time']
+            @server_update_timestamp = data['Time']
+          end
+        end
+      end
+
+      def self.xml_document_to_hash(document)
+        def self.xml_node_to_hash(xml)
+          return nil if xml.children.empty?
+          return xml.children.first.to_s if xml.children.count == 1 && xml.children.first.text?
+
+          # Append a sequential number to the name to prevent replacing items that should be in an array
+          child_number = 0
+          Hash[*(xml.children.map { |child| child_number += 1 ; ["#{child.name}::#{child_number}", xml_node_to_hash(child)] }.compact.flatten(1))]
+        end
+
+        intermediate_hash = xml_node_to_hash(document.root)
+
+        return Hash[*(intermediate_hash.group_by do |obj|
+          obj.first.gsub(/::\d+$/, '')
+        end.map do |key, value|
+          # Remove the 'key' entries
+          value = value.flatten(1).delete_if { |v| v.to_s =~ /#{key}::\d+/ }
+
+          # Remove the sequential number from the keys
+          value.map! do |element|
+            Hash[*(element.map do |ikey, ivalue|
+              [ikey.gsub(/::\d+$/, ''), ivalue]
+            end.flatten(1))]
+          end
+
+          # If there's only one entry, remove the array
+          value = value.first if value.count == 1
+
+          [key, value]
+        end.flatten(1))]
+      end
+
+      FileUtils.mkdir_p local_cache_base_path
+
+      # Load the server timestamp on startup
+      server_update_timestamp
+  end
+end