sjunkieex 0.0.1

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+*.swp
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/vcr_cassettes
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gem "turn"
+gem "ansi"
+
+# Specify your gem's dependencies in sjunkieex.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,18 @@
+
+(General Public License)
+
+Copyright (c) 2012 Philipp Böhm
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation in version 3 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+MA 02110-1301, USA.

data/README.md

@@ -0,0 +1,22 @@
+# Sjunkieex (Serienjunkies Extractor)
+
+Program that extracts links from serienjunkies.org for your series and
+searches for new episodes for existing series.
+
+## Installation
+
+Install it yourself with (add sudo for systemwide installation):
+
+    $ gem install sjunkieex
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+#!/usr/bin/env rake
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/test*.rb']
+  t.verbose = true
+end

data/bin/sjunkieex

@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby
+# -*- ruby -*-
+# encoding: UTF-8
+
+$LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'sjunkieex'
+require 'hashconfig'
+require 'optparse'
+require 'fileutils'
+
+# create program configuration dirs/files
+CONFIG_DIR  = File.join( File.expand_path("~"), ".sjunkieex" )
+CONFIG_FILE = File.join( CONFIG_DIR, "config.yml" )
+FileUtils.mkdir(CONFIG_DIR) unless File.directory?(CONFIG_DIR)
+
+###
+# configuration
+STANDARD_CONFIG = {
+  :hd_series       => ["SERIES_NOT_EXISTING_PLACEHOLDER"],
+  :index_directory => File.join(CONFIG_DIR, ".index/"),
+  :index_suffix    => "xml",
+  :hoster_id       => "nl",
+  :dump_links      => true,
+  :dump_file       => "/tmp/gsl_links.txt",
+}.merge(Sjunkieex::Interface::STANDARD_CONFIG)
+
+config = STANDARD_CONFIG.merge_with_serialized(CONFIG_FILE)
+
+###
+# option definition and handling
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename($PROGRAM_NAME)}"
+
+  opts.separator("")
+  opts.separator("Tool that extracts episodelinks from serienjunkies.org.")
+  opts.separator("")
+  opts.separator("  Options:")
+
+  # opts.on( "-i", "--ignore-seriesinfo",
+  #          "do not use the information from the infostore") do |opt|
+  #   config[:read_episode_info] = false
+  # end
+
+  opts.on( "-v", "--version",
+           "Outputs the version number.") do |opt|
+    puts Sjunkieex::VERSION
+    exit
+  end
+
+  opts.separator("")
+
+end.parse!
+
+fail "index directory #{ config[:index_directory]} does not exist" unless
+    File.directory?(config[:index_directory])
+
+
+glob_pattern = File.join(config[:index_directory] ,"*.#{config[:index_suffix]}")
+files = Dir[glob_pattern]
+
+series_index = Sjunkieex::SeriesIndex.new(files: files)
+
+fail "there is data for series existing" if series_index.empty?
+
+###
+# look for new series
+dump_links = []
+interface = Sjunkieex::Interface.new(series_index, config)
+
+interface.look_for_new_episodes.each do |link,series|
+    puts "\nLook for new episodes in '#{series}'"
+
+    links = interface.parse_series_page(series, link)
+    links.each do |identifier, link_data|
+        puts link_data[:episodedata]
+
+        hd = false
+        (hd = true) if config[:hd_series].include? series
+
+        ###
+        # select links, depending on wanted resolution
+        links = []
+        if hd
+            if link_data[:hd_1080p]
+                links = link_data[:hd_1080p]
+            elsif link_data[:hd_720p]
+                links = link_data[:hd_720p]
+            end
+        else
+            if link_data[:sd]
+                links = link_data[:sd]
+            end
+        end
+
+        if links.empty?
+            puts "there are no links in this resolution"
+            next
+        end
+
+        download_links = links.select do |link|
+            link.match(/\/f-\w+\/#{ config[:hoster_id] }_/)
+        end
+
+        if download_links.empty?
+            puts "there are no links for this hoster"
+            next
+        end
+
+        print download_links[0].to_yaml
+        dump_links << download_links[0]
+    end
+end
+
+###
+# dump links to file
+if config[:dump_links]
+    File.open(config[:dump_file], 'w') do |f|
+        dump_links.each { |l| f.write(l + "\n")}
+    end
+end

data/lib/sjunkieex/interface.rb

@@ -0,0 +1,147 @@
+require 'open-uri'
+require 'zlib'
+
+module Sjunkieex
+
+  class Interface
+
+    STANDARD_CONFIG = {
+          url: "http://serienjunkies.org",
+          german_only: true,
+          hd: false,
+          subbed_allowed: false,
+      }
+
+    attr_reader :options
+
+    def initialize(series_index, options = {})
+      @options = STANDARD_CONFIG.merge(options)
+      @index = series_index
+    end
+
+    # Public: Looks for new episodes on the homepage
+    #
+    # Returns a Hash of links for sites that should be visited
+    def look_for_new_episodes
+      links = Hash.new
+
+      doc = Nokogiri::XML(get_page_data(@options[:url]))
+      doc.css("div#content > div.post > div.post-content a").each do |link|
+        c = link.content
+
+        ####
+        # skip links that are not suitable
+        next unless is_link_useful?(c)
+
+        next unless @index.is_series_in_index?(c)
+
+        series_name = Sjunkieex::SeriesIndex.extract_seriesname(c)
+        next unless series_name
+
+        next if @index.episode_existing?(series_name, c)
+
+        href = link[:href]
+        next if links.include?(href)
+
+        links[href] = series_name
+      end
+
+      return links
+    end
+
+    # Public: parses a series page and extracts links
+    #
+    # series_name - the series name and the key in the index
+    # series_link - the link to the page
+    #
+    # Returns a hash indexed by series identifier
+    def parse_series_page(series_name, series_link)
+
+      link_data = Hash.new
+
+      doc = Nokogiri::XML(get_page_data(series_link))
+      doc.css("div#content > div.post > div.post-content p").each do |paragraph|
+
+        next if paragraph[:class]
+
+        episode_data = paragraph.css("strong:first-child").text
+        next unless is_link_useful?(episode_data)
+
+        next if @index.episode_existing?(series_name, episode_data)
+
+        if id = Sjunkieex::SeriesIndex.extract_episode_identifier(episode_data)
+
+          # classify episode resolution
+          resolution = :sd
+          (resolution = :hd_720p) if episode_data.match(/720[pi]/i)
+          (resolution = :hd_1080p) if episode_data.match(/1080[pi]/i)
+
+          # extract hoster links
+          episode_links = []
+          paragraph.css("a").each do |link|
+            episode_links << link[:href]
+          end
+
+          (link_data[id] = Hash.new) unless link_data[id]
+          link_data[id][resolution] = episode_links
+          link_data[id][:episodedata] = episode_data
+          link_data[id][:series] = series_name
+        end
+      end
+
+      return link_data
+    end
+
+    private
+
+    # Internal: check the link data against criterias
+    #
+    # link_data - data for the link
+    #
+    # Returns true if the link is useful or false if it can be skipped
+    def is_link_useful?(link_data)
+
+        return false unless link_data.match(/S\w+E\w+/i)
+
+        # skip links depending on language
+        if @options[:german_only]
+          return false unless link_data.match(/German/i)
+
+          unless @options[:subbed_allowed]
+            return false if link_data.match(/Subbed/i)
+          end
+        else
+          return false if link_data.match(/German/i)
+        end
+
+        true
+    end
+
+    # Internal: get a page and do some stuff if the page is gzip encoded
+    #
+    # link - the link that is fetched
+    #
+    # Returns the page content
+    def get_page_data(link)
+
+      body = nil
+
+      stream = open(link)
+      if stream.is_a? File
+        # file is a local file, has not methods below
+        body = stream.read
+      else
+        # file is web uri
+        if (stream.content_encoding.empty?)
+          body = stream.read
+        else
+          body = Zlib::GzipReader.new(stream).read
+        end
+      end
+
+      return body
+    end
+
+  end
+
+end

data/lib/sjunkieex/series_index.rb

@@ -0,0 +1,134 @@
+require 'nokogiri'
+
+module Sjunkieex
+
+  class SeriesIndex
+
+    attr_reader :options, :series_data
+
+    # Public: instantiate a new series_index
+    #
+    # options  - Options (default: {})
+    #                :files  - Array of series indizes
+    #
+    def initialize(options = {})
+      @options = {files: [], }.merge(options)
+
+      @series_data = Hash.new
+      @options[:files].each do |file|
+        @series_data.merge!(parse_file(file))
+      end
+
+    end
+
+    # Public: checks if there are entries in the index
+    #
+    # Returns true if there no entries loaded
+    def empty?
+      @series_data.length == 0
+    end
+
+    # Public: Check if a supplied episode is in the index
+    #
+    # series_name - Name of the series in the index
+    # episode_text - episode data
+    #
+    # Returns true if the episode is existing, false otherwise
+    def episode_existing?(series_name, episode_text)
+      if @series_data[series_name]
+
+          if id = SeriesIndex.extract_episode_identifier(episode_text)
+            if @series_data[series_name][id]
+              return true
+            end
+          end
+      end
+
+      return false
+    end
+
+    # Public: checks if the seriesname in the supplied data is in the index
+    #
+    # episode_text  - data that contains the episode information
+    #
+    # Returns true if the series is in the index, false otherwise
+    def is_series_in_index?(episode_text)
+
+      if series_name = SeriesIndex.extract_seriesname(episode_text)
+        if @series_data[series_name]
+          return true
+        end
+      end
+
+      return false
+    end
+
+    # Public: tries to extract the seriesname from supplied data
+    #
+    # data - data that holds the episode information
+    #
+    # Returns the seriesname or nil if there is no seriesname
+    def self.extract_seriesname(data)
+      if md = data.match(/(.*)S\d+E\d+/)
+        return md[1].gsub(/\./, " ").strip
+      end
+      nil
+    end
+
+    # Public: tries to extract the episode identifier from the episode data
+    #
+    # data - data that holds the episode information
+    #
+    # Returns the identifier xx_xx or nil if there is no identifier
+    def self.extract_episode_identifier(data)
+      if md = data.match(/S(\d+)E(\d+)/i)
+        return "%s_%s" % [md[1].to_i, md[2].to_i]
+      end
+      nil
+    end
+
+    private
+
+    # Internal: parse this file to a hash indexed by seriesname
+    #
+    #  file - path to the xml file
+    #
+    # Returns a Hash indexed by seriesname with Hashes as values
+    #
+    #     hash = {
+    #       "Chase": {
+    #          "1_1": "S01E01 - test.avi",
+    #       }
+    #     }
+    def parse_file(file)
+
+      series_data = Hash.new
+
+      content = File.open(file, "r").read
+      doc = Nokogiri::XML(content)
+
+      doc.css("serienindex > directory").each do |series_node|
+
+        title = series_node[:name]
+        next unless title && title.match(/\w+/)
+
+        series = Hash.new
+        series_node.css("file").each do |file_node|
+
+          filename = file_node[:name]
+          next unless filename
+
+          if id = SeriesIndex.extract_episode_identifier(filename)
+            series[id] = filename
+          end
+        end
+
+        series_data[title] = series
+      end
+
+      series_data
+    end
+
+  end
+
+end

data/lib/sjunkieex/version.rb

@@ -0,0 +1,3 @@
+module Sjunkieex
+  VERSION = "0.0.1"
+end

data/lib/sjunkieex.rb

@@ -0,0 +1,7 @@
+require "sjunkieex/version"
+require "sjunkieex/interface"
+require "sjunkieex/series_index"
+
+module Sjunkieex
+
+end

data/sjunkieex.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/sjunkieex/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Philipp Böhm"]
+  gem.email         = ["philipp@i77i.de"]
+  gem.description   = %q{Tool that extracts links from serienjunkies.org}
+  gem.summary       = %q{serienjunkies.org link extractor}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "sjunkieex"
+  gem.require_paths = ["lib"]
+  gem.version       = Sjunkieex::VERSION
+
+  gem.add_runtime_dependency(%q<nokogiri>, [">= 1.5"])
+  gem.add_runtime_dependency(%q<hashconfig>, [">= 0.0.1"])
+end