xxx_rename 0.0.1

data/lib/xxx_rename/site_clients/algolia_common.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "httparty"
+require "nokogiri"
+
+module XxxRename
+  module SiteClients
+    module AlgoliaCommon
+      CLIENT_KEY_JS_REGEX_1 = /var\sclient\s=\salgoliasearch\('(?<application_id>\w+)',\s'(?<api_key>[\w=]+)'\);/x.freeze
+      CLIENT_KEY_JS_REGEX_2 = /
+      window\.env\s*=\s*{
+        "api":{
+          "algolia":{
+            "applicationID":"(?<application_id>\w+)",
+            "apiKey":"(?<api_key>[\w=]+)"
+          }
+        }/x.freeze
+
+      ALGOLIA_RATE_LIMIT_MESSAGE = "Too many requests"
+      ALGOLIA_RATE_LIMIT_STATUS = 429
+      ALGOLIA_EXPIRY_MESSAGE = "\"validUntil\" parameter expired (less than current date)"
+      ALGOLIA_EXPIRY_STATUS = 400
+
+      AlgoliaParams = Struct.new(:application_id, :api_key, keyword_init: true)
+
+      def algolia_params!(site)
+        doc = Nokogiri::HTML HTTParty.get(site, headers: Constants::DEFAULT_HEADERS).parsed_response
+        js = doc.search("script").text
+        match = js.match(CLIENT_KEY_JS_REGEX_1) || js.match(CLIENT_KEY_JS_REGEX_2)
+        raise "Unable to fetch algolia credentials" if match.nil?
+
+        AlgoliaParams.new(application_id: match[:application_id], api_key: match[:api_key])
+      end
+
+      def actors_contained?(actors_from_file, actors_from_response)
+        #  All actors in actors_from_file should be contained in actors_from_response
+        get_actor = ->(hsh) { hsh[:name] }
+        actors_from_file_set = actors_from_file.map(&:normalize).to_set
+        actors_from_response_set = actors_from_response.map(&get_actor).map(&:normalize).to_set
+        actors_from_file_set.subset? actors_from_response_set
+      end
+
+      def date_released(resp)
+        Time.strptime(resp[:release_date].strip, "%Y-%m-%d")
+      end
+    end
+  end
+end

data/lib/xxx_rename/site_clients/algolia_v2.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require "algolia"
+
+require "xxx_rename/site_clients/algolia_common"
+
+module XxxRename
+  module SiteClients
+    class AlgoliaV2 < Base
+      include AlgoliaCommon
+
+      def client(refresh: false)
+        if refresh
+          XxxRename.logger.debug "Refreshing Algolia Token...".colorize(:blue)
+
+          @client = nil
+          @scenes_index = nil
+          @movies_index = nil
+          @actor_index = nil
+        end
+
+        @client = client! if @client.nil?
+
+        @client
+      end
+
+      def scenes_index
+        @scenes_index ||= client.init_index(self.class::SCENES_INDEX_NAME)
+      end
+
+      def movies_index
+        @movies_index ||= client.init_index(self.class::MOVIES_INDEX_NAME)
+      end
+
+      def actors_index
+        @actors_index ||= client.init_index(self.class::ACTORS_INDEX_NAME)
+      end
+
+      def fetch_scenes_from_api(str)
+        with_retry { scenes_index.search(str, default_query)&.[](:hits) }
+      end
+
+      def fetch_actor_from_api(str)
+        with_retry { actors_index.search(str)&.[](:hits) }
+      end
+
+      private
+
+      def client!
+        Algolia::Search::Client.new(algolia_config, logger: XxxRename.logger)
+      end
+
+      def algolia_config
+        params = algolia_params!(@site_url)
+        algolia_config = Algolia::Search::Config.new(application_id: params.application_id,
+                                                     api_key: params.api_key)
+        algolia_config.set_extra_header("Referer", @site_url)
+        algolia_config
+      end
+
+      def with_retry(current_attempt: 1, max_attempts: 5, &block)
+        if current_attempt > max_attempts
+          raise XxxRename::Errors::FatalError, "Retry exceeded #{self.class.name} ran exceeded retry attempts #{max_attempts}"
+        end
+
+        block.call
+      rescue Algolia::AlgoliaHttpError => e
+        case e.code
+        when 429
+          XxxRename.logger.error "[RATE LIMIT EXCEEDED] Sleeping for 3 minutes. Cancel to run the app at a different time."
+          6.times do |counter|
+            sleep(30)
+            XxxRename.logger.info "[SLEEP ELAPSED] #{counter * 30}s"
+          end
+        else
+          XxxRename.logger.error "#{e.class}: code:#{e.code} message:#{e.message}"
+        end
+        client(refresh: true)
+        with_retry(current_attempt: current_attempt + 1, max_attempts: max_attempts, &block)
+      end
+
+      def default_query
+        {
+          attributesToRetrieve: %w[clip_id title actors release_date description network_name movie_id movie_title],
+          hitsPerPage: 50
+        }
+      end
+
+      def default_facet_filters
+        %w[upcoming:0 content_tags:straight]
+      end
+
+      def find_matched_scene!(search_results, match)
+        scenes = search_results.reject { |x| x[:release_date].nil? }
+                               .uniq { |x| x[:clip_id] }
+                               .select { |x| actors_contained?(match.actors, x[:actors]) }
+                               .select { |x| x[:title].normalize.start_with?(match.title.normalize) }
+
+        raise Errors::NoMatchError.new(Errors::NoMatchError::ERR_NO_RESULT, match.title) if scenes.length != 1
+
+        make_scene_data(scenes.first)
+      end
+
+      def make_scene_data(scene)
+        hash = {}.tap do |h|
+          h[:female_actors] = female_actors(scene)
+          h[:male_actors] = male_actors(scene)
+          h[:actors] = female_actors(scene) + male_actors(scene)
+          h[:collection] = scene[:network_name]&.strip&.titleize
+          h[:collection_tag] = site_config.collection_tag
+          h[:title] = scene[:title]&.strip
+          h[:id] = scene[:clip_id].to_s
+          h[:date_released] = date_released(scene[:release_date])
+          movie_hash = movie_details(scene[:movie_id]) || nil
+          h[:movie] = movie_hash unless movie_hash.nil?
+        end
+
+        Data::SceneData.new(hash)
+      end
+
+      def find_movie(movie_id)
+        options = {
+          filters: "movie_id:#{movie_id}",
+          attributesToRetrieve: %w[movie_id title description date_created
+                                   studio_name directors network_name
+                                   url_title cover_path],
+          hitsPerPage: 1
+        }
+        with_retry { movies_index.search("", options)&.[](:hits)&.first }
+      end
+
+      def movie_details(movie_id)
+        return movies[movie_id] if movies.key?(movie_id)
+
+        movie = find_movie(movie_id)
+        return if movie.nil?
+
+        movie_hash = {}.tap do |h|
+          h[:name] = movie[:title]&.strip&.titleize
+          date = date_released(movie[:date_created])
+          h[:date] = date if date
+          h[:url] = URI.join(@site_url, "/en/movie/", "#{movie[:url_title]}/", (movie[:movie_id]).to_s).to_s
+          h[:front_image] = "#{self.class::CDN_BASE_URL}/movies#{movie[:cover_path]}_front_400x625.jpg?width=900&height=1272&format=webp"
+          h[:back_image] = "#{self.class::CDN_BASE_URL}/movies#{movie[:cover_path]}_back_400x625.jpg?width=900&height=1272&format=webp"
+          h[:studio] = movie[:network_name]&.strip
+          h[:synopsis] = movie[:description]
+        end
+
+        movies[movie_id] = movie_hash
+        movie_hash
+      end
+
+      def female_actors(resp)
+        resp[:actors]
+          .select { |actor| actor[:gender] == "female" }
+          .map { |hash| hash[:name] }
+          .map(&:strip)
+          .sort
+      end
+
+      def male_actors(resp)
+        resp[:actors]
+          .select { |actor| actor[:gender] == "male" }
+          .map { |hash| hash[:name] }
+          .map(&:strip)
+          .sort
+      end
+
+      def date_released(str, format = "%Y-%m-%d")
+        Time.strptime(str.strip, format)
+      rescue ArgumentError => e
+        XxxRename.logger.error "[DATE PARSING ERROR] #{e.message}"
+        nil
+      end
+
+      def movies
+        @movies ||= {}
+      end
+    end
+  end
+end

data/lib/xxx_rename/site_clients/babes.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "xxx_rename/site_clients/mg_premium"
+
+module XxxRename
+  module SiteClients
+    class Babes < MGPremium
+      site_client_name :babes
+
+      def initialize(config)
+        super(config, site_url: "https://www.babes.com")
+      end
+    end
+  end
+end

data/lib/xxx_rename/site_clients/base.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "httparty"
+require "xxx_rename/site_clients/configuration"
+require "xxx_rename/utils"
+
+module XxxRename
+  module SiteClients
+    class Base
+      include Utils
+      include SiteClients::Configuration
+
+      attr_reader :source_format, :config
+
+      # @param [XxxRename::Data::Config] config
+      def initialize(config)
+        @actors_helper = ActorsHelper.instance
+        @config = config
+        @source_format = site_config.file_source_format
+        return unless self.class.include?(HTTParty)
+
+        self.class.logger(XxxRename.logger, :debug)
+        self.class.headers(Constants::DEFAULT_HEADERS)
+      end
+
+      def search(_filename, **_opts)
+        raise "Not Implemented."
+      end
+
+      private
+
+      def match?(str1, str2)
+        str1.normalize == str2.normalize
+      end
+
+      def contains?(str1, str2)
+        str1.normalize.include?(str2.normalize)
+      end
+
+      def actors_hash(actors)
+        actors.each { |x| @actors_helper.auto_fetch! x }
+        female_actors = actors.select { |x| @actors_helper.female? x }
+        male_actors = actors.select { |x| @actors_helper.male? x }
+        resp = { female_actors: female_actors,
+                 male_actors: male_actors,
+                 actors: female_actors + male_actors }
+        diff = (resp[:female_actors] + resp[:male_actors]) - actors
+        raise "Actors #{diff.join(", ")} removed from response even after being processed." if diff.length > 1
+
+        resp
+      rescue XxxRename::Errors::UnprocessedEntity => e
+        XxxRename.logger.warn "Unable to fetch details of actor #{e.message}"
+        {
+          female_actors: [],
+          male_actors: [],
+          actors: actors
+        }
+      end
+    end
+  end
+end

data/lib/xxx_rename/site_clients/blacked.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "xxx_rename/site_clients/vixen_media"
+
+module XxxRename
+  module SiteClients
+    class Blacked < VixenMedia
+      base_uri "https://www.blacked.com"
+      site_client_name :blacked
+    end
+  end
+end

data/lib/xxx_rename/site_clients/blacked_raw.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "xxx_rename/site_clients/vixen_media"
+
+module XxxRename
+  module SiteClients
+    class BlackedRaw < VixenMedia
+      base_uri "https://www.blackedraw.com/"
+      site_client_name :blacked_raw
+    end
+  end
+end

data/lib/xxx_rename/site_clients/brazzers.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "xxx_rename/site_clients/mg_premium"
+
+module XxxRename
+  module SiteClients
+    class Brazzers < MGPremium
+      site_client_name :brazzers
+
+      def initialize(config)
+        super(config, site_url: "https://www.brazzers.com")
+      end
+    end
+  end
+end

data/lib/xxx_rename/site_clients/configuration.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module XxxRename
+  module SiteClients
+    module Configuration
+      def self.included(base)
+        base.extend ClassMethods
+
+        # The `ModuleInheritableAttributes` allows us to use some ruby magic
+        base.send :include, HTTParty::ModuleInheritableAttributes
+        base.send(:mattr_inheritable, :constants)
+        base.instance_variable_set("@constants", {})
+
+        base.class_eval do
+          def site_config
+            config.site.send(self.class.site_client_name.to_sym)
+          end
+
+          def site_client_datastore
+            @site_client_datastore ||=
+              begin
+                datastore_name = site_config.database.presence ? site_config.database : self.class.name.demodulize.underscore
+                qualified_name = datastore_name.end_with?(".store") ? datastore_name : "#{datastore_name}.store"
+                path = File.join(config.generated_files_dir, self.class.site_client_name.to_s)
+                FileUtils.mkpath(path)
+                store = Data::SceneDatastore.new(path, qualified_name).store
+                Data::SceneDatastoreQuery.new(store, config.mutex)
+              end
+          end
+
+          # @return [Hash]
+          def metadata
+            site_client_datastore.metadata
+          end
+
+          # @param [Hash] opts
+          # @return [Hash]
+          def update_metadata(**opts)
+            site_client_datastore.update_metadata(opts)
+          end
+        end
+      end
+
+      module ClassMethods
+        def site_client_name(name = nil)
+          return @constants[:site_client_name] if @constants.key?(:site_client_name)
+
+          raise XxxRename::Errors::FatalError, "#{self.name} did not set its :site_client_name" if name.nil?
+
+          @constants[:site_client_name] = name
+        end
+      end
+    end
+  end
+end

data/lib/xxx_rename/site_clients/digital_playground.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "xxx_rename/site_clients/mg_premium"
+
+module XxxRename
+  module SiteClients
+    class DigitalPlayground < MGPremium
+      site_client_name :digital_playground
+
+      def initialize(config)
+        super(config, site_url: "https://www.digitalplayground.com")
+      end
+    end
+  end
+end

data/lib/xxx_rename/site_clients/elegant_angel.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+require "xxx_rename/site_clients/query_generator/base"
+
+module XxxRename
+  module SiteClients
+    class ElegantAngel < Base
+      include HTTParty
+
+      base_uri "https://www.elegantangel.com"
+      site_client_name :elegant_angel
+      OLDEST_PROCESSABLE_MOVIE_YEAR = 2007
+      COLLECTION = "Elegant Angel"
+      MOVIES_ENDPOINT = "/streaming-elegant-angel-dvds-on-video.html?page=$page$"
+
+      def search(filename)
+        refresh_datastore(1) if datastore_refresh_required?
+        match = SiteClients::QueryGenerator::Base.generic_generate(filename, source_format)
+        lookup_in_datastore!(match)
+      end
+
+      def all_scenes_processed?
+        all_scenes_processed
+      end
+
+      def oldest_processable_date?
+        oldest_processable_date
+      end
+
+      private
+
+      def lookup_in_datastore!(match)
+        msg = "requires both movie title(%collection) and scene title(%title)"
+        raise Errors::NoMatchError.new(Errors::NoMatchError::ERR_NO_METADATA, msg) if match.nil?
+
+        raise Errors::NoMatchError.new(Errors::NoMatchError::ERR_NO_METADATA, msg) unless match.collection.presence && match.title.presence
+
+        index_key = site_client_datastore.generate_lookup_key(match.collection, match.title)
+        scene_data_key = site_client_datastore.find_by_key?(index_key)
+        raise Errors::NoMatchError.new(Errors::NoMatchError::ERR_NO_RESULT, index_key) if scene_data_key.nil?
+
+        scene_data = site_client_datastore.find_by_key?(scene_data_key)
+        raise Errors::NoMatchError.new(Errors::NoMatchError::ERR_NO_RESULT, scene_data_key) if scene_data.nil?
+
+        scene_data
+      end
+
+      def all_scenes_processed
+        @all_scenes_processed ||= false
+      end
+
+      def oldest_processable_date
+        @oldest_processable_date ||= false
+      end
+
+      def datastore_refresh_required?
+        if config.force_refresh_datastore
+          XxxRename.logger.info "#{"[FORCE REFRESH]".colorize(:green)} #{self.class.name}"
+          true
+        else
+          false
+        end
+      end
+
+      def refresh_datastore(page)
+        movie_links = movie_links(page)
+        @all_scenes_processed = true if movie_links.blank?
+
+        movie_links.map do |path|
+          movie_doc = doc(path)
+          movie_hash = movie_hash(movie_doc, path)
+          if movie_hash[:date].year < OLDEST_PROCESSABLE_MOVIE_YEAR
+            @oldest_processable_date = true
+            break
+          end
+          scenes = movie_scenes(movie_doc, movie_hash)
+          scenes.map { |scene_data| site_client_datastore.create!(scene_data, force: true) }
+        end
+
+        stop_processing? ? true : refresh_datastore(page + 1)
+      end
+
+      def stop_processing?
+        if all_scenes_processed?
+          XxxRename.logger.info "#{"[DATASTORE REFRESH COMPLETE]".colorize(:green)} #{self.class.site_client_name}"
+          true
+        elsif oldest_processable_date?
+          XxxRename.logger.info "#{"[OLDEST PROCESSABLE MOVIE REACHED]".colorize(:green)} #{self.class.site_client_name}"
+          true
+        else
+          false
+        end
+      end
+
+      def movie_links(page)
+        doc = doc(MOVIES_ENDPOINT.gsub("$page$", page.to_s))
+        doc.css(".item-grid .grid-item .boxcover")
+           .map { |x| x["href"] }
+           .uniq
+           .map { |x| x.gsub(self.class.base_uri, "") }
+      end
+
+      def movie_scenes(doc, movie_hash)
+        XxxRename.logger.info "#{"[PROCESSING MOVIE]".colorize(:green)} #{movie_hash[:name]}"
+
+        scenes(doc).map do |scene_doc|
+          next if scene_unavailable?(scene_doc)
+
+          hash = {}.tap do |h|
+            h[:actors] = scene_doc.css(".scene-performer-names a").map { |x| x.text&.strip }
+            h[:collection] = movie_hash[:name]
+            h[:collection_tag] = site_config.collection_tag
+            h[:title] = scene_doc.at('//a[@class="scene-title"]//h6/text()[last()]')&.text&.strip
+            h[:date_released] = movie_hash[:date]
+            scene_path = scene_doc.at('//a[@class="scene-title"]/@href').value
+            h[:scene_link] = URI.join(self.class.base_uri, scene_path).to_s
+            h[:movie] = movie_hash
+          end
+          XxxRename.logger.info "#{"[PROCESSING SCENE]".colorize(:green)} #{hash[:title]}"
+          Data::SceneData.new(hash)
+        end.compact
+      end
+
+      def scene_unavailable?(scene_doc)
+        # scene title is un-clickable
+        scene_doc.at('//a[@class="scene-title"]').nil? ||
+          # buying scene is disabled
+          scene_doc.at('//div[contains(@class, "scene-buy-options")]//button[contains(@class, "disabled")]').present? # scene title is un-clickable
+      end
+
+      def scenes(doc)
+        doc.xpath('//div[@id="scenes"]//div[@class="scene-details"]').presence ||
+          doc.xpath('//div[@id="scenes"]//div[@class="grid-item"]').presence ||
+          []
+      end
+
+      def movie_hash(doc, path)
+        image_css = doc.xpath('//div[@id="viewLargeBoxcoverCarousel"]//div[contains(@class, "carousel-item")]//img/@data-src')
+        date_str = doc.xpath('//div[contains(@class, "video-details")]' \
+                  '//div[@class="release-date"]' \
+                  '//span[contains(text(),"Released")]/../text()').text&.strip
+        synopsis = doc.xpath('//div[contains(@class, "video-details")]//div[@class="synopsis"]').text
+        {}.tap do |h|
+          h[:name] = doc.css(".video-title h1.description").text&.strip
+          h[:date] = date_released(date_str, "%b %d, %Y")
+          h[:url] = URI.join(self.class.base_uri, path).to_s
+          h[:front_image] = image_css.first.text
+          h[:back_image] = image_css.last.text if image_css.length == 2
+          h[:studio] = "Elegant Angel"
+          h[:synopsis] = synopsis if synopsis.presence
+        end
+      end
+
+      def date_released(str, format = "%Y-%m-%d")
+        Time.strptime(str.strip, format)
+      rescue ArgumentError => e
+        XxxRename.logger.error "[DATE PARSING ERROR] #{e.message}"
+        nil
+      end
+
+      def doc(path)
+        res = handle_response!(return_raw: true) { self.class.get(path) }
+        Nokogiri::HTML res.parsed_response
+      end
+    end
+  end
+end

data/lib/xxx_rename/site_clients/errors.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module XxxRename
+  module SiteClients
+    module Errors
+      class NoMatchError < StandardError
+        attr_reader :code, :data
+
+        ERR_NO_METADATA = 0
+        ERR_NO_RESULT = 1
+        ERR_NW_REDIRECT = 2
+        ERR_CUSTOM = 3
+
+        def initialize(code, data)
+          @code = code
+          @data = data
+          super(make_message)
+        end
+
+        def make_message
+          case code
+          when ERR_NO_RESULT then "No results from API #{"using query #{data}" if data}"
+          when ERR_NO_METADATA then "No metadata parsed from file"
+          when ERR_NW_REDIRECT then "Network redirect from search #{data}"
+          when ERR_CUSTOM then data || "No metadata parsed from file"
+          else "Unhandled error when trying to process file (Code #{code})"
+          end
+        end
+      end
+
+      class SiteClientUnavailableError < StandardError
+        def initialize(site)
+          super("Network #{site} unreachable. Check if site is accessible.")
+        end
+      end
+
+      class InvalidCredentialsError < StandardError
+        def initialize(site)
+          super("Missing/Invalid credentials provided for #{site}")
+        end
+      end
+
+      class SearchError < StandardError
+        attr_reader :entity, :request_options, :response_code, :response_body
+
+        def initialize(entity, object)
+          @entity = entity
+          @request_options = object[:request_options]
+          @response_code = object[:response_code]
+          @response_body = object[:response_body]
+          super("Network Error while fetching details for file #{@entity}")
+        end
+
+        def dump_error
+          File.open(File.join($pwd, "error_dump.txt"), "a") do |dump| # rubocop:disable Style/GlobalVars
+            dump << "--------ERROR BEGIN--------\n#{message}\n"
+            dump << "--------ENTITY--------\n#{@entity}\n"
+            dump << "--------REQUEST--------\n#{@request_options}\n"
+            dump << "--------CODE--------\n#{@response_code}\n"
+            dump << "--------BODY--------\n#{@response_body}\n"
+            dump << "--------ERROR END--------\n\n\n"
+          end
+        end
+      end
+
+      class APIError < StandardError
+        attr_reader :endpoint, :code, :body, :headers
+
+        def initialize(endpoint:, code:, body:, headers:)
+          @endpoint = endpoint
+          @code = code
+          @body = body
+          @headers = headers
+          super(message)
+        end
+
+        def fetch_error_message
+          case code
+          when 302 then "unexpected redirection to #{headers["location"]}"
+          else (body&.[]("error") || body&.[]("message") || body).to_s[0..150]
+          end
+        end
+
+        def message
+          "API Failure:\n" \
+          "\tURL: #{endpoint}\n" \
+          "\tRESPONSE CODE: #{code}\n" \
+          "\tERROR MESSAGE: #{fetch_error_message}"
+        end
+      end
+
+      class BadGatewayError < APIError; end
+      class BadRequestError < APIError; end
+      class ForbiddenError < APIError; end
+      class InternalServerError < APIError; end
+      class NotFoundError < APIError; end
+      class RedirectedError < APIError; end
+      class TooManyRequestsError < APIError; end
+      class UnauthorizedError < APIError; end
+      class UnhandledError < APIError; end
+    end
+  end
+end