serienrenamer 0.0.14 → 0.0.15

data/lib/serienrenamer/plugin/wikipedia.rb

@@ -0,0 +1,448 @@
+# encoding: UTF-8
+require 'media_wiki'
+
+module Serienrenamer
+  module Plugin
+
+    # This Plugin tries to extract the series
+    # information from wikipedia
+    #
+    # (by now only the german wikipedia)
+    class Wikipedia < Serienrenamer::Pluginbase
+
+      def self.plugin_name; "Wikipedia" end
+      def self.usable; true end
+      def self.priority; 30 end
+
+      @@WIKIPEDIA_URL = 'http://de.wikipedia.org/w/api.php'
+
+      # patterns used in this class
+      @@EPISODE_TABLE_PATTERN = /.*(?<table>\{\{Episodenlistentabelle.*\}\})\s*$/m
+      @@EPISODE_ENTRY_PATTERN = /\{\{Episodenlisteneintrag|S-Episode/
+      @@SERIES_SITE_TEST_PATTERN = /\{\{Infobox.Fernsehsendung.*\}\}/m
+      @@DISAMBIGUATION_TEST_PATTERN = /\{\{Begriffsklärung\}\}/m
+      @@CONTAINS_LINK_TO_EPISODE_LIST = /Hauptartikel.*(?<main>Liste.*?)[\]\}]+/
+      @@CONTAINS_INARTICLE_EPISODE_LIST = /\<div.*\>Staffel.(\d+).*\<\/div\>.*class=\"wikitable\".*titel/m
+      @@INPAGE_SEASON_SEPARATOR = /\<div.style=\"clear:both\;.class=\"NavFrame\"\>/
+      @@WIKITABLE_EXTRACT_PATTERN = /(\{\|.class=\"wikitable\".*\|\})\n/m
+      @@IS_ONE_LINE_EPISODE_LIST = /\|.*\|\|.*\|\|.*\|\|/m
+
+
+      # this method will be called from the main program
+      # with an Serienrenamer::Episode instance as parameter
+      #
+      # it returns an array of episode information
+      def self.generate_episode_information(episode)
+
+        raise ArgumentError, "Serienrenamer::Episode instance needed" unless
+          episode.is_a?(Serienrenamer::Episode)
+
+        return [] unless episode.series.match(/\w+/)
+
+        unless defined? @cached_data
+          @cached_data = Hash.new
+        end
+
+        wiki = MediaWiki::Gateway.new(@@WIKIPEDIA_URL)
+
+        if ! @cached_data.has_key?(episode.series)
+          # search for a series site in wikipedia
+          series_site = nil
+          tries = 3
+          search_pattern = episode.series
+          search_pattern_modified = false
+
+          begin
+            wiki.search(search_pattern, nil, 15).each do |title|
+              pagedata = wiki.get(title)
+              if is_series_main_page?(pagedata)
+                series_site = title
+                break
+              end
+            end
+
+            # modify the search term pattern so that it contains
+            # only the last word if the search_pattern contains
+            # more than one words
+            if series_site.nil? && ! search_pattern_modified
+              search_pattern = search_pattern.match(/(\w+)\s*$/)[1]
+              search_pattern_modified = true
+              raise EOFError if search_pattern # break out and retry
+            end
+          rescue MediaWiki::APIError => e
+            tries -= 1
+            retry if tries > 0
+          rescue EOFError => e
+            retry
+          end
+
+          return [] unless series_site
+
+          # look for a link to a list of episodes
+          pagedata = wiki.get(series_site)
+
+          if contains_link_to_episode_list?(pagedata)
+            mainarticle = pagedata.match(@@CONTAINS_LINK_TO_EPISODE_LIST)[:main]
+            if mainarticle
+              episodelist_page = wiki.get(mainarticle)
+              series = parse_episodelist_page_data(episodelist_page)
+
+              @cached_data[episode.series] = series
+            end
+
+          elsif contains_inarticle_episode_list?(pagedata)
+            series = parse_inarticle_episodelist_page_data(pagedata)
+            @cached_data[episode.series] = series
+
+          else
+            warn "no episode list found"
+            return []
+          end
+        end
+
+        episode_names = []
+
+        # tries to find an episodename in cached_data
+        # otherwise returns empty array
+        begin
+          series = @cached_data[episode.series]
+          episodename = series[episode.season][episode.episode]
+          if episodename.match(/\w+/)
+            episode_names.push(episodename)
+          end
+        rescue
+        end
+
+        return episode_names
+      end
+
+
+      # This method will extract season based information
+      # from a string that contains a wikipedia episodelist page
+      #
+      # returns an Array of Arrays with episode information
+      # where episode and season numbers are the indizes
+      def self.parse_episodelist_page_data(pagedata, debug=false)
+        raise ArgumentError, 'String with pagedata expected' unless
+        pagedata.is_a?(String)
+
+        series_data = []
+        is_season_table_following = false
+        season_number = nil
+
+        # split the wikipedia page by headings and process
+        # the following paragraph if the heading starts with
+        # 'Staffel'
+        pagedata.split(/(==.*)==/).each do |paragraph|
+          if paragraph.match(/^==.*Staffel/)
+            match = paragraph.match(/^==.*Staffel.(?<seasonnr>\d+)/)
+            if match
+              season_number = match[:seasonnr].to_i
+              is_season_table_following = true
+            end
+          elsif is_season_table_following
+            #
+            # extract season table from this paragraph
+            season = parse_season_table(paragraph)
+
+            series_data[season_number] = season
+            is_season_table_following = false
+          end
+        end
+
+        return series_data
+      end
+
+
+      # this method will be called with a wikipedia seasontable
+      # as parameter and will extract all episodes from this
+      # and returns that as an array where the episode number is
+      # the index
+      def self.parse_season_table(table)
+        raise ArgumentError, 'String with seasontable expected' unless
+        table.is_a?(String)
+
+        season_data = []
+
+        matched_table = table.match(@@EPISODE_TABLE_PATTERN)
+        if matched_table
+
+          # extract all episode entries that
+          # looks like the following
+          #
+          # {{Episodenlisteneintrag
+          # | NR_GES = 107
+          # | NR_ST = 1
+          # | OT = The Mastodon in the Room
+          # | DT = Die Rückkehr der Scheuklappen
+          # | ZF =
+          # | EA = {{dts|23|09|2010}}
+          # | EAD = {{dts|08|09|2011}}
+          # }}
+
+          episodes = matched_table[:table].split(@@EPISODE_ENTRY_PATTERN)
+          if episodes
+            episodes.each do |epi|
+
+              # build up a hash from the entry
+              infos = {}
+              epi.lines.each do |part|
+                parts = part.strip.match(/(?<key>\w+).=.(?<value>.*)$/)
+                if parts
+                  infos[parts[:key].strip] = parts[:value].strip
+                end
+              end
+
+              next unless infos.has_key?('NR_ST')
+
+              # extract useful information and
+              # add it to the array
+              epi_nr = infos['NR_ST'].to_i
+              next unless epi_nr
+
+              # TODO make the following variable
+              epi_name = infos['DT'].strip
+
+              # remove all html tags and all following
+              # text from the episode name and the bold
+              # syntax from mediawiki [[text]]
+              epi_name.gsub!(/<\/?[^>]*>.*/, "")
+              epi_name.gsub!(/[\[\[\]\]]/, "")
+              next unless epi_name.match(/\w+/)
+
+              season_data[epi_nr] = epi_name
+            end
+          end
+        end
+        return season_data
+      end
+
+
+      # This method will extract season based information
+      # from a string that contains a series page with an
+      # episodelist included
+      #
+      # returns an Array of Arrays with episode information
+      # where episode and season numbers are the indizes
+      def self.parse_inarticle_episodelist_page_data(pagedata, debug=false)
+        raise ArgumentError, 'String with pagedata expected' unless
+        pagedata.is_a?(String)
+
+        series_data = []
+
+        # look for a paragraph with an episodelist
+        episodelist_paragraph = pagedata.split(/==.*==/).select { |p|
+          contains_inarticle_episode_list?(p) }[0]
+
+          raise ArgumentError, 'no episodelist found' unless episodelist_paragraph
+
+          # iterate through all seasons in this episode table
+          episodelist_paragraph.split(@@INPAGE_SEASON_SEPARATOR).each do |season|
+            next unless contains_inarticle_episode_list?(season)
+
+            season_nr = season.match(@@CONTAINS_INARTICLE_EPISODE_LIST)[1].to_i
+
+            wikitable = season.match(@@WIKITABLE_EXTRACT_PATTERN)[1]
+
+            # we have to detect the type of the inarticle season page
+            # because there are two different kinds of table structures
+            # used in the german wikipedia
+            if self.is_episode_list_with_one_episode_per_line?(wikitable)
+              episodes = parse_inarticle_season_table_with_one_line(wikitable)
+            else
+              episodes = parse_inarticle_season_table(wikitable)
+            end
+
+            # HACK if a season is splitted into different parts
+            # eg. Flashpoint (2.1 and 2.2) than merge that if possible
+            if series_data[season_nr] != nil
+              series_data[season_nr].each_with_index do |item, index|
+                episodes[index] = item unless episodes[index]
+              end
+            end
+
+            series_data[season_nr] = episodes
+          end
+
+          return series_data
+      end
+
+
+      # this method will be called with a wikitable for a season
+      # as parameter and will extract all episodes from this
+      # and returns that as an array where the episode number is
+      # the index
+      #
+      # Example for an wikitable for episodes:
+      #
+      # {| class="wikitable" width="100%"
+      # |- vertical-align: top; text-align:center; "
+      # | width="15" | '''Nummer''' <br /><small>(Gesamt)<small>
+      # | width="15" | '''Nummer''' <br /><small>(Staffel)<small>
+      # ! width="250" | Originaltitel
+      # ! width="250" | Deutscher Titel
+      # ! width="180" | Erstausstrahlung<br /><small>(USA Network)</small>
+      # ! width="180" | Erstausstrahlung<br /><small>(RTL)</small>
+      # ! width="180" | Erstausstrahlung<br /><small>(SF zwei)</small>
+      # |-
+      # | bgcolor="#DFEEEF"| 01
+      # | 01
+      # | ''Pilot''
+      # | ''Auch Reiche sind nur Menschen''
+      # | 4. Mai 2009
+      # | 17. Mai 2011
+      # | 6. Juni 2011 (Teil 1)<br />13. Juni 2011 (Teil 2)
+      # |-
+      # |}
+      #
+      def self.parse_inarticle_season_table(table)
+        raise ArgumentError, 'String with seasontable expected' unless
+        table.is_a?(String)
+
+        season_data = []
+        episode_nr_line_nr   = nil
+        episode_name_line_nr = nil
+
+        table.split(/^\|\-.*$/).each do |tablerow|
+          tablerow.strip!
+
+          # skip invalid rows
+          lines = tablerow.lines.to_a
+          next unless lines.length >= 4
+
+          if tablerow.match(/width=\"\d+\"/)
+            # extract line numbers for needed data that
+            # are in the table header
+            lines.each_with_index do |item, index|
+              if item.match(/Nummer.*Staffel/i)
+                episode_nr_line_nr = index
+
+                # TODO make the following more variable
+              elsif item.match(/Deutscher.*Titel/i)
+                episode_name_line_nr = index
+              end
+            end
+          else
+            # extract episode information
+            if episode_nr_line_nr && episode_name_line_nr
+
+              md_nr = lines[episode_nr_line_nr].strip.match(/(\d+)/)
+              if md_nr
+                episode_nr = md_nr[1].to_i
+
+                md_name = lines[episode_name_line_nr].strip.match(/^\|.(.*)$/)
+                if md_name
+                  episode_name = md_name[1]
+                  episode_name.gsub!(/[\'\"\[\]]/, "")
+                  next unless episode_name.match(/\w+/)
+
+                  season_data[episode_nr] = episode_name.strip
+                end
+              end
+            end
+          end
+        end
+
+        return season_data
+      end
+
+
+      # this method will be called with a wikitable for a season
+      # as parameter and will extract all episodes from this
+      # and returns that as an array where the episode number is
+      # the index
+      #
+      # this method implements a special format that takes place in
+      # e.g. 'Prison Break' where an episode is not spread along several
+      # lines like in the method above
+      #
+      # Example for an wikitable for episodes:
+      #
+      #{| class="wikitable"
+      # |- style="color:#black; background-color:#006699"
+      # ! '''Episode''' !! '''Deutscher Titel''' !! '''Originaltitel''' !! '''Erstausstrahlung (DE)''' !! '''Erstausstrahlung (USA)'''
+      # |-
+      # |'''1''' (1-01) || Der große Plan || Pilot || 21. Juni 2007 || 29. August 2005
+      # |-
+      # |'''2''' (1-02) || Lügt Lincoln? || Allen || 21. Juni 2007 || 29. August 2005
+      # |-
+      # |'''3''' (1-03) || Vertrauenstest || Cell Test || 28. Juni 2007 || 5. September 2005
+      # |-
+      # |'''4''' (1-04) || Veronica steigt ein || Cute Poison || 28. Juni 2007 || 12. September 2005
+      #
+      def self.parse_inarticle_season_table_with_one_line(table)
+        raise ArgumentError, 'String with seasontable expected' unless
+        table.is_a?(String)
+
+        season_data = []
+        episode_nr_col   = nil
+        episode_name_col = nil
+
+        table.split(/^\|\-.*$/).each do |tablerow|
+
+          if tablerow.match(/!!.*!!.*!!/)
+            # extract column numbers from table header
+            tablerow.split(/!!/).each_with_index do |col,index|
+              episode_nr_col   = index if col.match(/Episode/i)
+              episode_name_col = index if col.match(/Deutsch.*Titel/i)
+            end
+
+          elsif tablerow.match(/\|\|.*\w+.*\|\|/)
+            tablerow.strip!
+            columns = tablerow.split(/\|\|/)
+
+            # the following cleanes up the column so that the following occurs
+            # " '''7''' (1-07) " => "7     1 07"
+            #
+            # we can now extract the last bunch of digits and this algorithm is
+            # some kind of format independent
+            dirty_episode_nr   = columns[episode_nr_col].gsub(/\D/, " ").strip
+            episode_nr = dirty_episode_nr.match(/(\d+)$/)[1]
+            next unless episode_nr
+
+            episode_name = columns[episode_name_col].strip
+            next unless episode_nr.match(/\w+/)
+
+            season_data[episode_nr.to_i] = episode_name
+          end
+        end
+
+        return season_data
+      end
+
+
+      # this method checks if the page is the main page
+      # for a series
+      #
+      # returns true if page contains the infobox that
+      # is typical for series pages in wikipedia
+      def self.is_series_main_page?(page)
+        page.match(@@SERIES_SITE_TEST_PATTERN) != nil
+      end
+
+      # check the site if it is a disambiguation site
+      #
+      # returns true if this site links to pages with
+      # themes with the same name
+      def self.is_disambiguation_site?(page)
+        page.match(@@DISAMBIGUATION_TEST_PATTERN) != nil
+      end
+
+      # test if the page contains a link to an article
+      # with an episode list
+      def self.contains_link_to_episode_list?(page)
+        page.match(@@CONTAINS_LINK_TO_EPISODE_LIST) != nil
+      end
+
+      # test if the page contains a episode list
+      def self.contains_inarticle_episode_list?(page)
+        page.match(@@CONTAINS_INARTICLE_EPISODE_LIST) != nil
+      end
+
+      # tests for the type of in article episode list
+      def self.is_episode_list_with_one_episode_per_line?(page)
+        page.match(@@IS_ONE_LINE_EPISODE_LIST) != nil
+      end
+    end
+  end
+end

data/lib/serienrenamer/plugin.rb

@@ -0,0 +1,26 @@
+module Serienrenamer
+
+  # Base plugin from which all exisiting plugins inherits
+  class Pluginbase
+
+    class << self; attr_reader :registered_plugins end
+    @registered_plugins = []
+
+    # if you inherit from this class, the child
+    # gets added to the "registered plugins" array
+    def self.inherited(child)
+      self.registered_plugins << child
+    end
+
+    def self.plugin_name; "PluginBase" end
+
+    def self.to_s
+      self.plugin_name
+    end
+  end
+
+  # include all existing plugins
+  module Plugin
+    Dir[File.dirname(__FILE__) + '/plugin/*.rb'].each {|file| require file }
+  end
+end

data/lib/serienrenamer/version.rb

@@ -1,3 +1,3 @@
 module Serienrenamer
-    VERSION = '0.0.14'
+    VERSION = '0.0.15'
 end

data/lib/serienrenamer.rb

@@ -1,28 +1,7 @@
-$:.unshift(File.dirname(__FILE__)) unless
-  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
-
+require 'serienrenamer/episode.rb'
+require 'serienrenamer/information_store.rb'
+require 'serienrenamer/version.rb'
+require 'serienrenamer/plugin'
 
 module Serienrenamer
-
-    require 'serienrenamer/episode.rb'
-    require 'serienrenamer/information_store.rb'
-    require 'serienrenamer/version.rb'
-
-    class Pluginbase
-
-        class << self; attr_reader :registered_plugins end
-            @registered_plugins = []
-
-        # if you inherit from this class, the child
-        # gets added to the "registered plugins" array
-        def self.inherited(child)
-            Pluginbase.registered_plugins << child
-        end
-
-        def self.plugin_name; "PluginBase" end
-
-        def self.to_s
-            self.plugin_name
-        end
-    end
 end

data/serienrenamer.gemspec

@@ -15,6 +15,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Serienrenamer::VERSION
 
+  gem.required_ruby_version = '>= 1.9.0'
   gem.add_runtime_dependency(%q<wlapi>, [">= 0.8.4"])
   gem.add_runtime_dependency(%q<savon>, ["~> 0.8.4"])
   gem.add_runtime_dependency(%q<mediawiki-gateway>, [">= 0.4.4"])

data/test/test_helper.rb

@@ -2,7 +2,7 @@ require 'stringio'
 require 'test/unit'
 require 'vcr'
 require File.dirname(__FILE__) + '/../lib/serienrenamer'
-require File.dirname(__FILE__) + '/../lib/plugin'
+require File.dirname(__FILE__) + '/../lib/serienrenamer/plugin'
 
 class TestHelper
 

data/test/test_plugin_episode_identifier.rb

@@ -9,7 +9,7 @@ class TestPluginEpisodeIdentifier < Test::Unit::TestCase
   }
 
   def setup
-    @plugin = Plugin::EpisodeIdentifier
+    @plugin = Serienrenamer::Plugin::EpisodeIdentifier
     TestHelper.create_test_files(@@files.values)
     TestHelper.cwd
   end

data/test/test_plugin_serienjunkies_de.rb

@@ -27,7 +27,7 @@ class TestPluginSerienjunkiesDe < Test::Unit::TestCase
 
   def test_seriespage_url_search
     VCR.use_cassette("sjunkie_de_#{method_name}") do
-      plugin = Plugin::SerienjunkiesDe
+      plugin = Serienrenamer::Plugin::SerienjunkiesDe
 
       misfits = plugin.find_link_to_series_page("Misfits")
       assert_equal("http://serienjunkies.de/misfits/", misfits)
@@ -46,7 +46,7 @@ class TestPluginSerienjunkiesDe < Test::Unit::TestCase
   def test_parse_seriespage
 
     VCR.use_cassette("sjunkie_de_#{method_name}") do
-      plugin = Plugin::SerienjunkiesDe
+      plugin = Serienrenamer::Plugin::SerienjunkiesDe
 
       seasons = plugin.parse_seriespage(
         "http://www.serienjunkies.de/royal-pains/")
@@ -72,7 +72,7 @@ class TestPluginSerienjunkiesDe < Test::Unit::TestCase
   def test_episode_information_generation
 
     VCR.use_cassette("sjunkie_de_#{method_name}") do
-      plugin = Plugin::SerienjunkiesDe
+      plugin = Serienrenamer::Plugin::SerienjunkiesDe
 
       flpo = Serienrenamer::Episode.new(@@files['flpo'])
       data = plugin.generate_episode_information(flpo)[0]

data/test/test_plugin_serienjunkies_feed.rb

@@ -21,7 +21,7 @@ class TestPluginSerienjunkiesFeed < Test::Unit::TestCase
 
   def setup
     unless defined? @feed
-      @feed = Plugin::SerienjunkiesOrgFeed
+      @feed = Serienrenamer::Plugin::SerienjunkiesOrgFeed
       @feed.feed_url = File.join( File.dirname(__FILE__),
                                  'serienjunkies_feed_sample.xml')
     end

data/test/{test_serienjunkies_org.rb → test_plugin_serienjunkies_org.rb}

@@ -21,7 +21,7 @@ class TestSerienjunkiesOrg < Test::Unit::TestCase
   def test_seriespage_url_search
 
     VCR.use_cassette("sjunkie_org_#{method_name}") do
-      plugin = Plugin::SerienjunkiesOrg
+      plugin = Serienrenamer::Plugin::SerienjunkiesOrg
 
       misfits = plugin.find_link_to_series_page("Misfits")
       assert_equal("http://serienjunkies.org/serie/misfits/", misfits)
@@ -37,7 +37,7 @@ class TestSerienjunkiesOrg < Test::Unit::TestCase
   def test_parse_seriespage
 
     VCR.use_cassette("sjunkie_org_#{method_name}") do
-      plugin = Plugin::SerienjunkiesOrg
+      plugin = Serienrenamer::Plugin::SerienjunkiesOrg
 
       seasons = plugin.parse_seriespage(
         "http://serienjunkies.org/royal-pains/")
@@ -58,7 +58,7 @@ class TestSerienjunkiesOrg < Test::Unit::TestCase
   def test_episode_information_generation
 
     VCR.use_cassette("sjunkie_org_#{method_name}") do
-      plugin = Plugin::SerienjunkiesOrg
+      plugin = Serienrenamer::Plugin::SerienjunkiesOrg
 
       flpo = Serienrenamer::Episode.new(@@files['flpo'])
       data = plugin.generate_episode_information(flpo)[0]

data/test/test_plugin_textfile.rb

@@ -40,7 +40,7 @@ class TestPluginTextfile < Test::Unit::TestCase
   def test_information_extraction
     VCR.use_cassette("textfile_#{method_name}") do
       how = Serienrenamer::Episode.new(@@directories['hmym'])
-      data = Plugin::Textfile.generate_episode_information(how)[0]
+      data = Serienrenamer::Plugin::Textfile.generate_episode_information(how)[0]
       how.add_episode_information(data, true)
       assert_equal("S07E05 - Die Exkursion.avi", how.to_s)
     end
@@ -49,7 +49,7 @@ class TestPluginTextfile < Test::Unit::TestCase
   def test_select_right_textfile
     VCR.use_cassette("textfile_#{method_name}") do
       how = Serienrenamer::Episode.new(@@directories['hmmg'])
-      data = Plugin::Textfile.generate_episode_information(how)[0]
+      data = Serienrenamer::Plugin::Textfile.generate_episode_information(how)[0]
       how.add_episode_information(data, true)
       assert_equal("S07E11 - Plan B.avi", how.to_s)
     end
@@ -58,7 +58,7 @@ class TestPluginTextfile < Test::Unit::TestCase
   def test_information_extraction_with_directory_parameter
     VCR.use_cassette("textfile_#{method_name}") do
       how = @@directories['hmym']
-      data = Plugin::Textfile.generate_episode_information(how)[0]
+      data = Serienrenamer::Plugin::Textfile.generate_episode_information(how)[0]
       assert_not_nil(data)
     end
   end