suby 0.3.1 → 0.4.0

data/bin/suby

@@ -13,6 +13,10 @@ option_parser = OptionParser.new do |opts|
     options[:lang] = lang
   end
 
+  opts.on '-f', '--force', 'Force subtitles download even if already exists' do |lang|
+    options[:force] = true
+  end
+
   opts.on '-h', '--help', 'Show usage' do
     puts opts
     exit

data/lib/suby/downloader/addic7ed.rb

@@ -2,6 +2,8 @@ module Suby
   class Downloader::Addic7ed < Downloader
     SITE = 'www.addic7ed.com'
     FORMAT = :file
+    SUBTITLE_TYPES = [:tvshow]
+
     LANG_IDS = {
       en:  1, es:  5, it:  7, fr:  8, pt: 10, de: 11, ca: 12, eu: 13, cs: 14,
       gl: 15, tr: 16, nl: 17, sv: 18, ru: 19, hu: 20, pl: 21, sl: 22, he: 23,
@@ -29,7 +31,7 @@ module Suby
       body.strip!
       raise NotFoundError, "show/season/episode not found" if body.empty?
       if body.include? FILTER_IGNORED
-        raise NotFoundError, "no subtitle available"
+        raise NotFoundError, "no subtitles available"
       end
       body
     end

data/lib/suby/downloader/opensubtitles.rb

@@ -0,0 +1,83 @@
+module Suby
+  # Based on https://github.com/byroot/ruby-osdb/blob/master/lib/osdb/server.rb
+  class Downloader::OpenSubtitles < Downloader
+    SITE = 'api.opensubtitles.org'
+    FORMAT = :gz
+    XMLRPC_PATH = '/xml-rpc'
+    SUBTITLE_TYPES = [:tvshow, :movie, :unknown]
+
+    USERNAME = ''
+    PASSWORD = ''
+    LOGIN_LANGUAGE = 'eng'
+    USER_AGENT = 'Suby v0.4'
+
+    SEARCH_QUERIES_ORDER = [:hash, :name] #There is also search using imdbid but i dont think it usefull as it
+                                          #returns subtitles for many different versions
+
+    # OpenSubtitles needs ISO 639-22B language codes for subtitles search
+    # See http://www.opensubtitles.org/addons/export_languages.php
+    # and http://en.wikipedia.org/wiki/List_of_ISO_639-2_codes
+    LANG_MAPPING = {
+      ar: "ara", bg: "bul", bn: "ben", br: "bre", bs: "bos", ca: "cat", cs: "cze", da: "dan", de: "ger", el: "ell",
+      en: "eng", eo: "epo", es: "spa", et: "est", eu: "baq", fa: "per", fi: "fin", fr: "fre", gl: "glg", he: "heb",
+      hi: "hin", hr: "hrv", hu: "hun", hy: "arm", id: "ind", is: "ice", it: "ita", ja: "jpn", ka: "geo", kk: "kaz",
+      km: "khm", ko: "kor", lb: "ltz", lt: "lit", lv: "lav", mk: "mac", mn: "mon", ms: "may", nl: "dut", no: "nor",
+      oc: "oci", pb: "pob", pl: "pol", pt: "por", ro: "rum", ru: "rus", si: "sin", sk: "slo", sl: "slv", sq: "alb",
+      sr: "scc", sv: "swe", sw: "swa", th: "tha", tl: "tgl", tr: "tur", uk: "ukr", ur: "urd", vi: "vie", zh: "chi"
+    }
+    LANG_MAPPING.default = 'all'
+
+    def download_url
+      SEARCH_QUERIES_ORDER.find(lambda { raise NotFoundError, "no subtitles available" }) { |type|
+        if subs = search_subtitles(search_query(type))['data']
+          @type = type
+          break subs
+        end
+      }.first['SubDownloadLink']
+    end
+
+    def search_subtitles(query)
+      return {} unless query
+      query = [query] unless query.kind_of? Array
+      xmlrpc.call('SearchSubtitles', token, query)
+    end
+
+    def token
+      @token ||= login
+    end
+
+    def login
+      response = xmlrpc.call('LogIn', USERNAME, PASSWORD, LOGIN_LANGUAGE, USER_AGENT)
+      unless response['status'] == '200 OK'
+        raise DownloaderError, "Failed to login with #{USERNAME}:#{PASSWORD}. " +
+                               "Server return code: #{response['status']}"
+      end
+      response['token']
+    end
+
+    def search_query(type = :hash)
+      return nil unless query = send("search_query_by_#{type}")
+      query.merge(sublanguageid: language(lang))
+    end
+
+    def search_query_by_hash
+      { moviehash: MovieHasher.compute_hash(file), moviebytesize: file.size.to_s } if file.exist?
+    end
+
+    def search_query_by_name
+      season && episode ? { query: show, season: season, episode: episode } : { query: file.base.to_s }
+    end
+
+    def search_query_by_imdbid
+      { imdbid: imdbid } if imdbid
+    end
+
+    def language(lang)
+      LANG_MAPPING[lang.to_sym]
+    end
+
+    def success_message
+      "Found by #{@type}"
+    end
+  end
+end

data/lib/suby/downloader/tvsubtitles.rb

@@ -3,6 +3,7 @@ module Suby
     SITE = 'www.tvsubtitles.net'
     FORMAT = :zip
     SEARCH_URL = '/search.php'
+    SUBTITLE_TYPES = [:tvshow]
 
     # cache
     SHOW_URLS = {}
@@ -75,7 +76,7 @@ module Suby
         a = subtitles.css('div.left_articles a').find { |a|
           a.name == 'a' and a[:href].start_with?('/subtitle')
         }
-        raise NotFoundError, "no subtitle available" unless a
+        raise NotFoundError, "no subtitles available" unless a
         url = a[:href]
         raise 'invalid subtitle url' unless url =~ /^\/subtitle-(\d+)\.html/
         url
@@ -83,8 +84,7 @@ module Suby
     end
 
     def download_url
-      @download_url ||= URI.escape '/' +
-                      get_redirection(subtitles_url.sub('subtitle', 'download'))
+      URI.escape '/' + get_redirection(subtitles_url.sub('subtitle', 'download'))
     end
   end
 end

data/lib/suby/downloader.rb

@@ -1,6 +1,9 @@
 require 'net/http'
 require 'cgi/util'
 require 'nokogiri'
+require 'xmlrpc/client'
+require 'zlib'
+require 'stringio'
 
 module Suby
   class Downloader
@@ -9,22 +12,21 @@ module Suby
       DOWNLOADERS << downloader
     end
 
-    attr_reader :show, :season, :episode, :file, :lang
+    attr_reader :show, :season, :episode, :video_data, :file, :lang
 
     def initialize(file, *args)
       @file = file
       @lang = (args.last || 'en').to_sym
-      case args.size
-      when 0..1
-        @show, @season, @episode = FilenameParser.parse(file)
-      when 3..4
-        @show, @season, @episode = args
-      else
-        raise ArgumentError, "wrong number of arguments: #{args.size+1} for " +
-                             "(file, [show, season, episode], [lang])"
+      @video_data = FilenameParser.parse(file)
+      if video_data[:type] == :tvshow
+        @show, @season, @episode = video_data.values_at(:show, :season, :episode)
       end
     end
 
+    def support_video_type?
+      self.class::SUBTITLE_TYPES.include? video_data[:type]
+    end
+
     def to_s
       self.class.name.sub(/^.+::/, '')
     end
@@ -33,6 +35,10 @@ module Suby
       @http ||= Net::HTTP.new(self.class::SITE).start
     end
 
+    def xmlrpc
+      @xmlrpc ||= XMLRPC::Client.new(self.class::SITE, self.class::XMLRPC_PATH)
+    end
+
     def get(path, initheader = {}, parse_response = true)
       response = http.get(path, initheader)
       if parse_response
@@ -85,13 +91,22 @@ module Suby
       format = self.class::FORMAT
       case format
       when :file
-        sub_name(contents).write contents
+        # nothing special to do
+      when :gz
+        begin
+          gz = Zlib::GzipReader.new(StringIO.new(contents))
+          contents = gz.read
+        ensure
+          gz.close if gz
+        end
       when :zip
         TEMP_ARCHIVE.write contents
-        Suby.extract_sub_from_archive(TEMP_ARCHIVE, format, file)
+        Suby.extract_sub_from_archive(TEMP_ARCHIVE, format, TEMP_SUBTITLES)
+        contents = TEMP_SUBTITLES.read
       else
         raise "unknown subtitles format: #{format}"
       end
+      sub_name(contents).write encode contents
     end
 
     def sub_name(contents)
@@ -105,11 +120,54 @@ module Suby
         'sub'
       end
     end
+
+    def imdbid
+      @imdbid ||= begin
+        nfo_file = find_nfo_file
+        convert_to_utf8_from_latin1(nfo_file.read)[%r!imdb\.[^/]+/title/tt(\d+)!i, 1] if nfo_file
+      end
+    end
+
+    def find_nfo_file
+      @file.dir.children.find { |file| file.ext == "nfo" }
+    end
+
+    def convert_to_utf8_from_latin1(content)
+      if content.valid_encoding?
+        content
+      else
+        enc = content.encoding
+        if content.force_encoding("ISO-8859-1").valid_encoding?
+          yield if block_given?
+          content.encode("UTF-8")
+        else
+          # restore original encoding
+          subtitles.force_encoding(enc)
+        end
+      end
+    end
+
+    def success_message
+      "Found"
+    end
+
+    def encode(subtitles)
+      if @lang == :fr
+        convert_to_utf8_from_latin1(subtitles) do
+          def self.success_message
+            "#{super} (transcoded from ISO-8859-1)"
+          end
+        end
+      else
+        subtitles
+      end
+    end
   end
 end
 
 # Defines downloader order
 %w[
+    opensubtitles
     tvsubtitles
     addic7ed
   ].each { |downloader| require_relative "downloader/#{downloader}" }

data/lib/suby/filename_parser.rb

@@ -4,7 +4,7 @@ module Suby
 
     # from tvnamer @ ab2c6c, with author's agreement, adapted
     # See https://github.com/dbr/tvnamer/blob/master/tvnamer/config_defaults.py
-    FILENAME_PATTERNS = [
+    TVSHOW_PATTERNS = [
       # foo.s0101
       /^(?<show>.+?)
       [ \._\-]
@@ -68,22 +68,19 @@ module Suby
       (?<season>[0-9]{1})
       (?<episode>[0-9]{2})
       [\._ -][^\/]*$/x,
-
-      # foo.0103*
-      /^(?<show>.+)
-      [ \._\-]
-      (?<season>[0-9]{2})
-      (?<episode>[0-9]{2,3})
-      [\._ -][^\/]*$/x
     ]
+    MOVIE_PATTERN = /^(?<movie>.*)[.\[( ](?<year>(?:19|20)\d{2})/
 
     def parse(file)
       filename = file.basename.to_s
-      found = FILENAME_PATTERNS.find { |pattern|
-        pattern =~ filename
-      }
-      raise "Wrong file format (#{file})" unless found
-      [clean_show_name($~[:show]), $~[:season].to_i, $~[:episode].to_i]
+      if TVSHOW_PATTERNS.find { |pattern| pattern.match(filename) }
+        m = $~
+        { type: :tvshow, show: clean_show_name(m[:show]), season: m[:season].to_i, episode: m[:episode].to_i }
+      elsif m = MOVIE_PATTERN.match(filename)
+        { type: :movie, name: clean_show_name(m[:movie]), year: m[:year].to_i }
+      else
+        { type: :unknown, name: filename }
+      end
     end
 
     # from https://github.com/dbr/tvnamer/blob/master/tvnamer/utils.py#L78-95

data/lib/suby/movie_hasher.rb

@@ -0,0 +1,31 @@
+module Suby
+  # from http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes
+  module MovieHasher
+
+    CHUNK_SIZE = 64 * 1024 # in bytes
+    MASK64 = 0xffffffffffffffff # 2^64 - 1
+
+    def self.compute_hash(file)
+      filesize = file.size
+      hash = filesize
+
+      # Read 64 kbytes, divide up into 64 bits and add each
+      # to hash. Do for beginning and end of file.
+      file.open('rb') do |f|
+        # Q = unsigned long long = 64 bit
+        f.read(CHUNK_SIZE).unpack("Q*").each do |n|
+          hash = (hash + n) & MASK64
+        end
+
+        f.seek([0, filesize - CHUNK_SIZE].max, IO::SEEK_SET)
+
+        # And again for the end of the file
+        f.read(CHUNK_SIZE).unpack("Q*").each do |n|
+          hash = (hash + n) & MASK64
+        end
+      end
+
+      "%016x" % hash
+    end
+  end
+end

data/lib/suby.rb

@@ -9,24 +9,29 @@ module Suby
   DownloaderError = Class.new StandardError
 
   SUB_EXTENSIONS = %w[srt sub]
-  TEMP_ARCHIVE = Path('__archive__')
+  TMPDIR = Path.tmpdir
+  TEMP_ARCHIVE = TMPDIR / 'archive'
+  TEMP_SUBTITLES = TMPDIR / 'subtitles'
 
   class << self
     include Interface
 
     def download_subtitles(files, options = {})
+      Zip.options[:on_exists_proc] = options[:force]
       files.each { |file|
         file = Path(file)
         if file.dir?
           download_subtitles(file.children, options)
         elsif SUB_EXTENSIONS.include?(file.ext)
           # ignore already downloaded subtitles
-        elsif SUB_EXTENSIONS.any? { |ext| f = file.sub_ext(ext) and f.exist? and !f.empty? }
+        elsif !options[:force] and SUB_EXTENSIONS.any? { |ext| f = file.sub_ext(ext) and f.exist? and !f.empty? }
           puts "Skipping: #{file}"
         elsif !file.exist? or video?(file)
           download_subtitles_for_file(file, options)
         end
       }
+    ensure
+      TMPDIR.rm_rf
     end
 
     def video?(file)
@@ -35,10 +40,9 @@ module Suby
 
     def download_subtitles_for_file(file, options)
       begin
-        show, season, episode = FilenameParser.parse(file)
         puts file
         success = Downloader::DOWNLOADERS.find { |downloader_class|
-          try_downloader(downloader_class.new(file, show, season, episode, options[:lang]))
+          try_downloader(downloader_class.new(file, options[:lang]))
         }
         error "\nNo downloader could find subtitles for #{file}" unless success
       rescue
@@ -49,6 +53,7 @@ module Suby
     end
 
     def try_downloader(downloader)
+      return false unless downloader.support_video_type?
       begin
         print "  #{downloader.to_s.ljust(20)}"
         downloader.download
@@ -59,7 +64,7 @@ module Suby
         error "Error: #{error.message}"
         false
       else
-        success "Found"
+        success downloader.success_message
         true
       end
     end
@@ -72,8 +77,7 @@ module Suby
             entry.to_s =~ /\.#{Regexp.union SUB_EXTENSIONS}$/
           }
           raise "no subtitles in #{archive}" unless sub
-          name = file.sub_ext(Path(sub).ext)
-          sub.extract(name.to_s)
+          sub.extract(file.to_s)
         }
       else
         raise "unknown archive type (#{archive})"

data/suby.gemspec

@@ -16,5 +16,5 @@ Gem::Specification.new do |s|
   s.add_dependency 'term-ansicolor'
   s.add_dependency 'mime-types', '>= 1.19'
 
-  s.version = '0.3.1'
+  s.version = '0.4.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: suby
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-08 00:00:00.000000000 Z
+date: 2012-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: path
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &2156184680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,15 +21,10 @@ dependencies:
         version: 1.3.0
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.3.0
+  version_requirements: *2156184680
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &2156184300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -37,15 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *2156184300
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &2156183760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -53,15 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *2156183760
 - !ruby/object:Gem::Dependency
   name: term-ansicolor
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &2156183140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -69,15 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *2156183140
 - !ruby/object:Gem::Dependency
   name: mime-types
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &2156182480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -85,12 +65,7 @@ dependencies:
         version: '1.19'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '1.19'
+  version_requirements: *2156182480
 description: Find and download subtitles
 email: eregontp@gmail.com
 executables:
@@ -100,10 +75,12 @@ extra_rdoc_files: []
 files:
 - bin/suby
 - lib/suby/downloader/addic7ed.rb
+- lib/suby/downloader/opensubtitles.rb
 - lib/suby/downloader/tvsubtitles.rb
 - lib/suby/downloader.rb
 - lib/suby/filename_parser.rb
 - lib/suby/interface.rb
+- lib/suby/movie_hasher.rb
 - lib/suby.rb
 - .gitignore
 - README.md
@@ -128,9 +105,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
 summary: Subtitles' downloader
 test_files: []
-has_rdoc: