subfinder 0.0.1

data/fixtures/vcr_cassettes/wrong_download_url.yml

@@ -0,0 +1,112 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://subscene.com/subtitles/english-text/MGNU9jG42CkOL8vcxVwLzwaKhIvuPAE0Bb7OqO62gY8-mFppv1EKkdjJVrTFiRBfXA2sxFE5E8je3FtYz11K6jGRckMYNf6S5LznnuUB5aV_PjsPghQ
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - rest-client/2.0.2 (darwin18 x86_64) ruby/2.3.7p456
+      Host:
+      - subscene.com
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Fri, 25 Jan 2019 13:12:49 GMT
+      Content-Type:
+      - text/html; charset=utf-8
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Set-Cookie:
+      - __cfduid=d54bf78588c18d70a094eac3fc7e5ce8a1548421969; expires=Sat, 25-Jan-20
+        13:12:49 GMT; path=/; domain=.subscene.com; HttpOnly; Secure
+      Cache-Control:
+      - private
+      Vary:
+      - Accept-Encoding
+      Expect-Ct:
+      - max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"
+      Server:
+      - cloudflare
+      Cf-Ray:
+      - 49eafe5f2f01730d-AMS
+      Content-Encoding:
+      - gzip
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        H4sIAAAAAAAAA+wZa3MTOfKzqeI/NHNVa+fW9tgJYUNim+MRgpeAsyTA5jiK
+        kmfaM0pmpImk8QM2//1Kj3nYaxao2m93yYeRWv1Sqx9Se3Dv2eTpxeXZMcQq
+        TUZ37wz0FxLCoqGHzBvdvaNhSMLR3TuNQYqKQBATIVENvVzNOgce+NUSIykO
+        vTnFRcaF8iDgTCFTQ29BQxUPQ5zTADtmUtDJQNBMQUgU6QQzIlcsGDZnJJHY
+        BLXKcOgpXCr/isyJRfVAimDo+f41ZhmqbsBTN7yS/iNJFX6i4bC/640GvqUw
+        chRVCY7O86kMkCF04BzFHAUcC8HFwLfLd+809IYbg3udzgc6g0TB+Bgefqxr
+        aqTHSmXy0Pe1tfZlTOfdiPMowYCHaDSSc+YrkbNri9K9khv63PuALKSzj52O
+        mSaUXYPAZOgFhHFGA5J4EAucVbKk093wR622T1mIy0f7vd6RRtrAOTzo6bnZ
+        mfSRRQmVccdY89XJ67cPr07u7z69npwezIPlu8Xp5wV5GY/n+dnj496T6S+T
+        m8mD3ejyoJM+z7J5//jldXj16ztx8Zy+eTL7/fGuXD4/3j8+uMK95+ryc7//
+        8sHVyZvg+tXl69mD8/3Tz4zlb5/sk3efzq7kWRT/Vpx3tVGJRASx506ZZFlC
+        A6IoZz7PkNnVEK3JKGc/L9PSJDWMrgGbTQ694njXhDmSGZnTgLMuDbjnFIi5
+        UEGuQMMLPWhKIvSXHQvzK5eo8wqkfDQfvvu8t+xMeo/nk87V6bve77un9ycn
+        +4vX/474+fMzgpfXeH6c8+P5p7f9QqRaJShjROX5JrZMfNU9y7/SrKOT85MX
+        Dy7x5HRF8yf0AMmzh2+Wv2fsFB++7z3P+un49LdlsH98ueh7X4mTur+Vu3Ci
+        vkJx906jsaAs5IsuZ8bFYAiznAX6AKC1A180xu2R5lRnfvfOwHdJYjDl4cpK
+        NHkDheE6YGQONBx6My7ytMPInEbmsI3QQZ5YrISab2NANpzfkHXXQsAbPddA
+        OfCJJfYNtaUvGFWc/Bc8Rf8ZEdcXMabojfQQzLjgULBoDHyj0MBnZK6/hfK5
+        RPE13bfJ3Bq6JAh4zpSf8Igyb3SqP8AFvMGISoXiW9qYvYbUapTwiHsbNvO9
+        0QDTKtftdXsw8DEdVZYK6dyM7t4pOblwNAI1LEiIlENvRhOForZTp4DbbCXU
+        IvoYUlXG49OYsAjBsoaClYPaqdOpOjy3W62aURMsdMZFCsQ44tCr5TXz8dxm
+        Go0BZVle92/PFaUbz+zyxis2Zhf97ZQxDUNkBW1iaZMN2jlJchx6FRPDZpor
+        xZnjI/NpSpWnz8IoCufGFAPfYhXmHGj/TkfFli0n53dOZEro98bKmrdNBV9I
+        LDX3YPTEQDai5q/45etxl2cJJ6HvweitGW1y2nDXhk0NNg+U3uZuBqVeQYJE
+        zOjSqzu4W5sSxlB4YJLn0Esp68RIo1gd7vZ62dK5f5FNHf2aIe0tpl8wNJnN
+        G63fAOL+yEVDIZYvi/vP3ugxA5sPeRDkQmAIi5gmCJngAUpJWQQrngsQeJOj
+        VN2BH+9pftnoIqYSYiJhisgg4VGEIcy4gFmucoGAS5JSZvJJd+BnJoUK8Edg
+        P2sTw/CcC7EyHFSMQFnA2RwZRRZgew0bygxwgSQtmJeG0f/w/7//6b8twcaV
+        4mkRhmtRtVY6agFWlRAa4pSIToIz5Y2q0CuzXEniQnLGuaoSw2hTF6GDvBbe
+        DZep/W3IAc9WdYJ6ObRlebfX2+/s9voPq8AYyIywgsMchTRVvfXOjuB+t7cz
+        8DVOUZE3C+jAJblGdQ+oXQTWFFQLqopS2th6TXAY9gHhVPwU8NTbYNGZ8STh
+        i44tIt7oX3Xk8vpQV3P9drSmYWkm/XTyE4xI0qFM1yN3xznVIBhXoEpCWTy2
+        syz2FawVEG/01CZ/eIeCztylH84EjwRJf5j5RnUyu9DVhQTKSiKBglz+MN8r
+        DDG0+r6VKOHXYm5ejjTArSyL20ujVv7WA8UdpcylKkp6Y5A5aqkEZ9HoIkY4
+        SwhDBVQCz4WEDixiFAhEICwQIk5ZdO/RwHcUVqRl856q2NSGKOFTkkDGszyx
+        RpacCF2rFF8QEUooBD6EKU0SjZEhzxIs+cJ0VSLt9vZ71YIpg0xBgnNMJPBZ
+        iZciURUeYWG5EhIqVrorIPM0s65UogkExhU4u5BpgsCZ3jskNKUKQ0AiVNwt
+        4t5ZrIy/WgDWz7Xxnqgg1sVZ5VM8LCOjdvI15D95gaHrTtG/ulkev371kuDT
+        t4/UcLeX7h1Ib+QOST8t1uLuO7kvFouu08w470Lr+mg+xBd9+fDk6c3ZS17K
+        mARI2N8pZPLghKYn//78gBx4o/fxypgfiQLKJAZKPvpRYU5WwBcyo4IEK/dM
+        e1oCtnMsY6YKmm11xhWgqofSaLSqtyltg2wDb0PUBtEG0obUPVcbDfqheWLa
+        M48ZSVaKBnIyvcJANT/CEMQR0A9Cj8znjz/+/ODVkvRi98ZhdW803oePO90s
+        l3GLiChPkSm5Y7Fv2xYrgSH04Z/AcAHPiMLWzhEQGILsBgKJwuMENVmL77Qt
+        YWoWI1RuRT5ZXZDoNUmxxXc+9D4eAemaFplmrCdS6GF0BGk3IzocX+sGlD5A
+        oZ7gjAtsGUsY9rc7Lfu4b0PIA6NxG5rWns02NO3x2UZWhxSmsm/WcnYlNWpE
+        mjtHrppEpNW0+2m2odUcvx1PmvAzNCeP3aAz3puMzej+eGJB494vFjKejN2g
+        N/7FYk/Gu5PmTldglpAAWz71I9oGz6uBeAHS+teyv1HKqSSRhVrVjESoO5Kl
+        wvU23LcbIoYfnUFL8aybcFes7g2hGJc+Ukfo6qiAYWnn9QWr5G3h0lXLRT+n
+        tnVcGg2J6oKmyHPV2lh1Dur7MJage4eguxQl1A3MDlYZ8hkscLoH94ZDaOYs
+        xBllGDYrR69jRuQv8NaMjHNkxodeoSIpkdd6fInaV740GWdjplDY13vzEPq3
+        7pysFYpvGTx7vV5v5wh8H/ZAYsBZKIHMFArQhwnaStaCR6UJfR8uBAmugYSd
+        acKDa8qicukb9nVY37Kx72+3soWXw5r9FL9GBsPtJtS4jTkRYFQqHeUmR7E6
+        xwQDxUXLo0x2SSinKxuWXhl1hlzLIiH89BOQsEv1K/nFxavTKk7+I/3Ihk6C
+        LFIxDIfQW1Pg+47bom4/8cehsff3HbhldFuNq+FttbEfcAHfv/1zWOtXbvm0
+        KEtH8QqpRb9zjE+KTDlPdGrehJg0b3QvQDblf4FZksv4EJTIEez21vut9cue
+        uUuaM7GdpGre6R9435eIdBGq6GSlax1Yr0r9A2v0tZ8dapfRbwks3ohOTMD5
+        NdUnIZGpT9xc37QWX8BLUUoSoXcI3nvUgQEWWYLigEzqNseK5xChMvfSKUoF
+        uMxQmK5Fcc9b4FTf3r02eCGVKZVSczzhCqi6p6G6R8RecWEk6S/oh4pZoexa
+        A7f8DPKVd80/nIqaWpmG8KFmE8Wq457AsNnn/obJXP/+3Mykv2avbkrZn38D
+        +g52xSWOhLJbvkh8nXv+RnYklH5/k58NI9fNH/ju98H/AgAA//8DAKMOVJcx
+        HAAA
+    http_version: 
+  recorded_at: Fri, 25 Jan 2019 13:12:50 GMT
+recorded_with: VCR 4.0.0

data/lib/subfinder.rb

@@ -0,0 +1,19 @@
+require 'json'
+require 'colorize'
+require 'logger'
+require 'optparse'
+require 'zip'
+
+require_relative 'subfinder/version'
+require_relative 'subfinder/subtitle'
+require_relative 'subfinder/config'
+require_relative 'subfinder/main'
+require_relative 'subfinder/logger'
+
+require_relative 'subfinder/parser/args'
+require_relative 'subfinder/parser/files'
+require_relative 'subfinder/parser/subscene'
+require_relative 'subfinder/parser/download'
+
+module Subfinder
+end

data/lib/subfinder/config.rb

@@ -0,0 +1,8 @@
+module Subfinder
+  class Config
+    @video_formats = %w[.avi .mp4 .mkv]
+    class << self
+      attr_accessor :subtitles, :language, :url, :working_dir, :video_formats, :debug
+    end
+  end
+end

data/lib/subfinder/logger.rb

@@ -0,0 +1,22 @@
+module Subfinder
+  class Logger < ::Logger
+    class << self
+      def setting
+        logger = Logger.new STDOUT
+        logger.level = Logger::INFO
+        logger.formatter = proc do |_severity, _datetime, _progname, msg|
+          "#{msg}\n"
+        end
+        @logger = logger
+      end
+
+      def info(str)
+        @logger.info(str)
+      end
+
+      def debug(str)
+        @logger.info(str) if Config.debug
+      end
+    end
+  end
+end

data/lib/subfinder/main.rb

@@ -0,0 +1,20 @@
+module Subfinder
+  class Main
+    class << self
+      def run(args)
+        Subfinder::Logger.setting
+        Logger.info welcome_message
+        Subfinder::Parser::Args.new args
+        Subfinder::Subtitle.new.match
+      end
+
+      def welcome_message
+        '''
+
+        welcome to subfinder Gem!
+
+        '''.green
+      end
+    end
+  end
+end

data/lib/subfinder/parser/args.rb

@@ -0,0 +1,41 @@
+module Subfinder
+  module Parser
+    class Args
+      def initialize(args)
+        @args = args
+        parse_args
+        validate_required_args
+      end
+
+      def parse_args
+        opts = OptionParser.new
+        opts.banner = usage_msg
+        opts.separator ''
+        opts.separator 'options:'
+        opts.on('-u', '--url url', 'Set subscene url page') { |url| Config.url = url }
+        opts.on('-l', '--language language', 'Set subtitle language') { |language| Config.language = language }
+        opts.on('-v', '--version', 'Show the Compare Crypto version') { puts("compare-crypto #{CompareCrypto::VERSION}"); exit }
+        opts.on('-h', '--help', 'Show this message') { puts(opts); exit }
+        opts.on('-d', '--debug', 'debug') { Config.debug = true }
+
+        opts.parse!(@args)
+      end
+
+      private
+
+      def validate_required_args
+        Config.language ||= 'en'
+        Config.working_dir = Dir.pwd
+      end
+
+      def usage_msg
+        usage = '
+          Usage:
+            subfinder [options]
+            See https://github.com/sizief/subfinder for more information. Emm, nothing there really, just source code, if you want to improve the code or report a bug visit the page.
+        '
+        usage
+      end
+    end
+  end
+end

data/lib/subfinder/parser/download.rb

@@ -0,0 +1,35 @@
+require 'rest-client'
+module Subfinder
+  module Parser
+    class Download
+      def initialize(url)
+        @url = url
+      end
+
+      def save
+        res = RestClient.get @url
+        return false unless response_is_healthy? res
+
+        file_name = res.headers[:content_disposition].split('=')[1]
+        File.write("#{Config.working_dir}/#{file_name}", res.body)
+        Logger.info "Downloaded to #{Config.working_dir}/#{file_name}"
+        true
+      rescue StandardError => e
+        Logger.info "Error when downloading '#{@url}'\n Error message: #{e}\n".red
+        false
+      end
+
+      def response_is_healthy?(res)
+        if res.code != 200
+          Logger.info "Error when downloading '#{@url}'\n Error message: resposnse code is #{res.code}\n".red
+          false
+        elsif res.body.include? 'An error occurred while processing your request.'
+          Logger.info "Error when downloading '#{@url}'\n Error message:  'An error occurred while processing your request.'\n".red
+          false
+        else
+          true
+        end
+      end
+    end
+  end
+end

data/lib/subfinder/parser/files.rb

@@ -0,0 +1,40 @@
+module Subfinder
+  module Parser
+    class Files
+      class << self
+        def list
+          prepare_file_list
+          Dir["#{Config.working_dir}/*"]
+        end
+
+        def prepare_file_list
+          files = Dir["#{Config.working_dir}/*"]
+          compressed_files = files.select { |file| ['.zip', '.rar'].include? File.extname(file) }
+          extract_all compressed_files
+          move_compressed_files compressed_files
+        end
+
+        def extract_all(array_list)
+          array_list.each do |file|
+            Zip::File.open(file) do |zip_file|
+              zip_file.each do |entry|
+                file_path = Config.working_dir + '/' + entry.name
+                unless File.exist? file_path
+                  Logger.info "Extracting #{entry.name}"
+                  entry.extract(file_path)
+                end
+              end
+            end
+          end
+        end
+
+        def move_compressed_files(compressed_files)
+          unless compressed_files.empty?
+            Dir.mkdir(Config.working_dir + '/compressed') unless File.exist?(Config.working_dir + '/compressed')
+            compressed_files.each { |f| File.rename(f, File.dirname(f) + '/compressed/' + File.basename(f)) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/subfinder/parser/subscene.rb

@@ -0,0 +1,93 @@
+require 'nokogiri'
+require 'open-uri'
+
+module Subfinder
+  module Parser
+    class Subscene
+      attr_reader :link
+      DOMAIN = 'https://subscene.com'.freeze
+
+      def initialize(file_name)
+        @file_name = file_name
+      end
+
+      def get
+        target_uri = find_link(create_links_list(open_online_document(Config.url)))
+        Subfinder::Parser::Download.new(find_download_link(open_online_document(DOMAIN + target_uri))).save
+      end
+
+      def open_online_document(url)
+        open url
+      rescue StandardError => e
+        Logger.info "Error when connecting to '#{url}'\n Error message: #{e}\n".red
+        abort('Check your internet connection or VPN and try again')
+      end
+
+      def find_download_link(page)
+        doc = Nokogiri::HTML(page)
+        download_link = nil
+        doc.css('.download').each do |link|
+          download_link = link.xpath('./a/@href').to_s.strip # link
+        end
+        Logger.debug "download_link: #{download_link}"
+        DOMAIN + download_link
+      end
+
+      def find_link(links)
+        max_match_point = 0
+        winner = ''
+
+        links.each do |link|
+          next if link[0] != Config.language
+
+          match_point = match_point_for link[1]
+          if match_point > max_match_point
+            max_match_point = match_point
+            winner = link[2]
+          end
+        end
+        Logger.debug "winner is #{winner}"
+        winner
+      end
+
+      def match_point_for(target)
+        point = 0
+        file_name_array = @file_name.split('.')
+        # next we want to know if name of the sub is seprataed by . or space
+        target_array = target.split('.').size > target.split(' ').size ? target.split('.') : target.split(' ')
+        target_array = target_array.map(&:downcase)
+
+        file_name_array.each do |word|
+          point += 1 if target_array.include? word.downcase
+        end
+        (point * 100) / file_name_array.size
+      end
+
+      def create_links_list(page)
+        array = []
+        doc = Nokogiri::HTML(page)
+        doc.xpath('//table//tr').each do |link|
+          next if link.xpath('./td/a/span/text()')[0].nil? # language
+
+          array << [convert_to_code(link.xpath('./td/a/span/text()')[0].to_s.strip), # language
+                    link.xpath('./td/a/span/text()')[1].to_s.strip, # name
+                    link.xpath('./td/a/@href')[0].to_s.strip] # link
+        end
+        Logger.debug "array_list: #{array}"
+        array
+      end
+
+      def convert_to_code(language)
+        case language
+        when 'Farsi/Persian'
+          'fa'
+        when 'English'
+          'en'
+        else
+          language
+        end
+        # TODO: add other languages codes
+      end
+    end
+  end
+end

data/lib/subfinder/subtitle.rb

@@ -0,0 +1,59 @@
+module Subfinder
+  class Subtitle
+    def initialize
+      @success = 0
+      @failure = 0
+    end
+
+    def match
+      Subfinder::Parser::Files.list.each do |video_file|
+        next unless Subfinder::Config.video_formats.include? File.extname(video_file)
+        next if subtitle_exists? video_file
+
+        if episode_number(video_file).nil?
+          Logger.info "Can not find the episode and season number for: #{File.basename(video_file)}".red
+          @failure += 1
+          next
+        else
+          find_subtitle_for video_file
+        end
+      end
+      Logger.info "Sub added: #{@success}, Sub not found: #{@failure}, Total file proccessed: #{@failure + @success}".green
+    end
+
+    private
+
+    def episode_number(video_file)
+      video_file.scan(/[a-zA-Z]+\d+\d+[a-zA-Z]+\d+\d/).first.downcase
+    end
+
+    def subtitle_exists?(video_file)
+      return true if Subfinder::Parser::Files.list.include? video_file.split('.')[0..-2].join('.') + '.srt'
+    end
+
+    def find_subtitle_for(video_file)
+      subtitles = Subfinder::Parser::Files.list.select { |item| item[/#{episode_number(video_file)}/i] && item[/.srt/] }
+      if subtitles.empty?
+        Logger.info "Subtitle for #{File.basename(video_file)} is not exists on disk "
+        subscene = Subfinder::Parser::Subscene.new(video_file)
+        if !Config.url.nil? && subscene.get
+          Subfinder::Parser::Files.prepare_file_list
+          Logger.info "Subtitle downloaded for #{File.basename(video_file)}"
+        else
+          Logger.info "Subtitle can not found on Subscene for #{File.basename(video_file)}".red
+          Logger.info 'Please provide a url with "-u" siwtch'.red if Config.url.nil?
+          @failure += 1
+          return
+        end
+      end
+      rename_subtitle video_file
+    end
+
+    def rename_subtitle(video_file)
+      subtitles = Subfinder::Parser::Files.list.select { |item| item[/#{episode_number(video_file)}/i] && item[/.srt/] }
+      File.rename(subtitles.first, File.dirname(subtitles.first) + '/' + File.basename(video_file).split('.')[0..-2].join('.') + '.srt')
+      Logger.info "Subtitle renamed for: #{File.basename(video_file)}"
+      @success += 1
+    end
+  end
+end