disavow_tool 0.1.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7ec191f824e770810f284c9893491f27468279a136bbfc075f8319e51f84c7a
-  data.tar.gz: 63626b8f6505c63e13930da9fb16b25d543256230ad149475d8edbc229a4b459
+  metadata.gz: 8db2e508792ce319b2f11c7e4dd43a3e7d31b875a0c3d80aa97828b600e34e96
+  data.tar.gz: 868088592ece6710d0326ce9b586c79e072f083cb9bdc2060998c1649552bc87
 SHA512:
-  metadata.gz: 0e77f7322716892366f2c22c1af5c7d6a31a8d50e088322bca3b1620aedab22b66440eee819590baac8cdd2a32e5100735f96c7565ec7203b4751cad8622878b
-  data.tar.gz: 751b98ce7205bb45e3eb472c1e76d8262aedca76e2d18774faacd0985a8385ceaad7e3e447f7e8c558927a05cc0fc29dfdc7a1c22ffced73244589faf7027cfa
+  metadata.gz: a7e6e2ab72198469ff1ec3cfa68497fbefcfec8eba7746ad1c2e17a087a045f2a8c1539de12642c04841256e83958095a20d0c0bc06b625940baf52c796b0546
+  data.tar.gz: a4da05637f583bf94cd3b4eb83c234297adf18954339757529e8ace255a008bf6674ed2fa2c80184dcf1002fc624b31695290b7161cfa304e18b58ce74901776

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    disavow_tool (0.1.1)
+    disavow_tool (0.3.0)
       activesupport
       colorize
+      nokogiri
 
 GEM
   remote: https://rubygems.org/
@@ -18,7 +19,10 @@ GEM
     diff-lcs (1.3)
     i18n (1.6.0)
       concurrent-ruby (~> 1.0)
+    mini_portile2 (2.4.0)
     minitest (5.11.3)
+    nokogiri (1.10.3)
+      mini_portile2 (~> 2.4.0)
     rake (10.5.0)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)

data/bin/disavow_tool_dev

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative '../lib/disavow_tool'
+
+DisavowTool.run

data/disavow_tool.gemspec

@@ -36,4 +36,5 @@ Gem::Specification.new do |spec|
   # Dependencies
   spec.add_dependency "colorize"
   spec.add_dependency "activesupport"
+  spec.add_dependency "nokogiri"
 end

data/lib/disavow_tool.rb

@@ -11,8 +11,8 @@ require_relative 'disavow_tool/white_list.rb'
 require_relative 'disavow_tool/imported_links.rb'
 
 module DisavowTool
-  def self.run
 
+  def self.run
     puts "Importing new links".blue if OPTIONS.verbose
     imported_links = ImportedLinks.new
 
@@ -23,7 +23,10 @@ module DisavowTool
       puts "Importing Whitelist links".blue if OPTIONS.verbose
       white_list = WhiteList.new
       puts "Cleagning imported links already in whitelist".blue if OPTIONS.verbose
-      imported_links.remove_known_links(white_list)
+      imported_links.remove_known_links(white_list.links)
+      puts "Cleagning imported links with a domain existingin in whitelist".blue if OPTIONS.verbose
+      p white_list.domains
+      imported_links.remove_known_links_for_domain(white_list.domains)
     end
 
 

data/lib/disavow_tool/command_options.rb

@@ -13,13 +13,14 @@ module DisavowTool
       options.whitelist  = false
       options.verbose = false
       options.hardcore_verbose = false
+      options.network_requests = true
 
       opt_parser = OptionParser.new do |opts|
           opts.banner = "Usage: disavow.rb [options] --disavow FILE --import file_1,file_2,file_3 [--whitelist file1,file2,file3]"
           opts.separator ""
           opts.separator "Requited options:"
-          opts.on("-d","--disavow FILE", "Disavow file as exported from Google Search Console") do |file|
-            options.disavow_file = file
+          opts.on("-d","--disavow file_1,file_2", Array, "Disavow files as exported from Google Search Console") do |file|
+            options.disavow_files = file
           end
           opts.on("-i","--import file_1,file_2", Array, "List of URLS to analyse. The file must have one URL per line") do |file|
             options.import_files = file
@@ -36,6 +37,11 @@ module DisavowTool
           opts.on("-v", "--verbose", "Vervose mode") do
             options.verbose = true
           end
+
+          opts.on("-t", "--no-titles", "Don't request tittles from websites thus making the command faster") do
+            options.network_requests = false
+          end
+
           opts.on("-V", "--hardcore-vervose", "Print out even your mama") do
             options.hardcore_verbose = true
             options.verbose = true            # Hardcose verbose includes regular verbose
@@ -60,7 +66,7 @@ module DisavowTool
     end
 
     def check_arguments(options)
-      raise "You must to specify one disallow file" if options.disavow_file.blank?
+      raise "You must to specify one disallow file" if options.disavow_files.blank?
       raise "You must to specify one import file" if options.import_files.blank?
       if options.whitelist
         raise "You need to specify at least one white list file" if options.whitelist_files.blank?

data/lib/disavow_tool/config.rb

@@ -1,4 +1,5 @@
 module DisavowTool
   EXPORT_PATH = "exports/"
   IMPORT_PATH = "test/"
+  SECONDS_TITTLE_REQUEST = 5
 end

data/lib/disavow_tool/disavow_list.rb

@@ -1,101 +1,33 @@
 require_relative 'list'
+require_relative 'domain_and_url'
 require 'uri'
 
 module DisavowTool
   class DisavowList < List
-    attr_accessor :disavowed_links, :disavowed_domains
-    alias_method :links, :disavowed_links
-    alias_method :domains, :disavowed_domains
+    include DomainAndUrl
 
-    def initialize(import_file=nil)
-      import_file = import_file || OPTIONS.disavow_file
-      @disavowed_domains = Set.new
-      @disavowed_links = Set.new
-      super(import_file)
-    end
-
-    def clean_line!(line)
-      case domain_or_url(line)
-      when :domain
-        @disavowed_domains << remove_domain_prefix(line)
-      when :url
-        @disavowed_links << line
-      end
-    end
-
-    def finished_import_hook
-      @original_disavowed_domains = @disavowed_domains.clone
-      @original_disavowed_links = @disavowed_links.clone
-    end
-
-    def add_domain(domain)
-      super(domain, @disavowed_domains)
-    end
-
-    def add_url(url, list=nil)
-      super(url, @disavowed_links)
-    end
-
-    def add_domain_from_url(url)
-      domain = URI.parse(URI.escape(url)).host
-      add_domain(domain)
-      p domain
-      return domain
+    def initialize(import_files=nil)
+      import_files = import_files || OPTIONS.disavow_files
+      super(import_files)
     end
 
     def import_message(domain)
       "Importing #{is_url?(domain).to_s} #{remove_domain_prefix(domain)} into Disavow list"
     end
     def add_url_message(url)
-      "+++ Inserting #{is_url?(url).to_s} #{url} in Disavow"
+      "+++ Inserting #{is_url?(url).to_s} #{url.on_yellow} in Disavow"
     end
     def message_sumary_imported; "Disavowed elements imported" end
     def mensaje_sumary_before_export; "Disavow elements before exporting" end
 
-    def summary
-      puts "Disavowed URLs:".light_blue
-      super(@disavowed_links, @original_disavowed_links)
-      puts "Disavowed Domains:".light_blue
-      super(@disavowed_domains, @original_disavowed_domains)
-    end
+    def message_sumary_links_imported; "Disavowed URLs:" end
+    def message_sumary_domains_imported; "Disavowed Domains:" end
 
     def export_write(file)
-      file.puts "# Domains"
-      file.puts @disavowed_domains.to_a
-      puts "Writing #{@disavowed_domains.count} Disavowed domains".blue if @verbose
-      file.puts "# urls"
-      file.puts @disavowed_links.to_a
-      puts "Writing #{@disavowed_links.count} Disavowed URLS".blue if @verbose
-
-    end
-
-    :private
-    def domain_or_url(line)
-      if( /^domain/.match(line))
-        return :domain
-      elsif( /^http/.match(line) )
-        return :url
-      else
-        raise "Error parsing Disavow file"
-      end
-    end
-
-    def is_url?(link)
-      if( link.match(/^http(s)?\:/) )
-        :url
-      else
-        :domain
-      end
+      file.puts "# Disavow"
+      super(file)
+      puts "Writing #{total_elements} elements into the Disavow file".blue if @verbose
     end
 
-    def remove_domain_prefix(domain)
-      domain.gsub(/^domain\:/, '')
-    end
-    def remove_domain_prefix!(domain)
-      domain.gsub!(/^domain\:/, '')
-    end
-
-    def add_domain_prefix
-    end
   end
 end

data/lib/disavow_tool/domain_and_url.rb

@@ -0,0 +1,93 @@
+module DisavowTool
+	module DomainAndUrl
+		attr_accessor :links, :domains
+
+		def initialize(import_files)
+			@domains = Set.new
+			@links = Set.new
+			p "DomainAndUrl"
+			super(import_files)
+		end
+
+		def clean_line!(line)
+			case domain_or_url(line)
+			when :domain
+				@domains << remove_domain_prefix(line)
+			when :url
+				@links << line
+			end
+		end
+
+		def finished_import_hook
+			@original_domains = @domains.clone
+			@original_links = @links.clone
+		end
+
+		def add_domain(domain)
+			super(domain, @domains)
+		end
+
+		def add_url(url, list=nil)
+			super(url, @links)
+		end
+
+		def add_domain_from_url(url)
+			domain = URI.parse(URI.escape(url)).host
+			add_domain(domain)
+			p domain
+			return domain
+		end
+
+		def export_write(file)
+			file.puts "# Domains"
+			add_domain_prefix
+			file.puts @domains.to_a
+			puts "Writing #{@domains.count} domains".blue if @verbose
+			file.puts "# urls"
+			file.puts @links.to_a
+			puts "Writing #{@links.count} URLS".blue if @verbose
+		end
+
+		def summary
+			puts message_sumary_links_imported.green
+			super(@links, @original_links)
+			puts message_sumary_domains_imported.green
+			super(@domains, @original_domains)
+		end
+
+		:private
+		def domain_or_url(line)
+			if( /^domain/.match(line))
+				return :domain
+			elsif( /^http/.match(line) )
+				return :url
+			else
+				raise "Error parsing Disavow file"
+			end
+		end
+
+		def is_url?(link)
+			unless( link.match(/^http(s)?\:/) )
+				:domain
+			else
+				:url
+			end
+		end
+
+		def remove_domain_prefix(domain)
+			domain.gsub(/^domain\:/, '')
+		end
+		def remove_domain_prefix!(domain)
+			domain.gsub!(/^domain\:/, '')
+		end
+
+		def add_domain_prefix
+			@domains.collect!{|domain| domain="domain:" + domain}
+		end
+
+		def total_elements
+			@domains.count + @links.count
+		end
+
+	end
+end

data/lib/disavow_tool/imported_links.rb

@@ -1,4 +1,8 @@
 require_relative 'list'
+require 'nokogiri'
+require 'open-uri'
+require 'timeout'
+
 module DisavowTool
   class ImportedLinks < List
 
@@ -17,18 +21,34 @@ module DisavowTool
     def analyse(disavowed, white_list)
        "Ready to delete analize #{@list.count} remaining links"
        @list.each do |url|
-         puts "#{"*"*100}\n* Analysing url: #{url.on_green}\n#{"*"*100}"
-         display_menu()
+         puts "#{"*"*100}\n*"
+         puts "* Analysing url: #{url.on_green}"
+         if OPTIONS.network_requests
+           print "* "+ "Obtaining website's title...\r".red.blink
+           puts "* Website title: #{website_title(url)}".ljust(100)
+         end
+         puts "* URls with this same domain: #{urls_with_same_domain(url)}"
+         puts "*\n#{"*"*100}"
+         puts menu()
          input = $stdin.getch
          input = $stdin.getch if open_browser_option(input, url)
          case input
               when "w"
+                raise "Command run with no whitelist option" if OPTIONS.whitelist == false
                 white_list.add_url url
                 self.delete_url url
+              when "W"
+                raise "Command run with no whitelist option" if OPTIONS.whitelist == false
+                domain = white_list.add_domain_from_url(url)
+                self.delete_url url
+                puts "Attempting to remove URLs with the domain #{domain} from imported links to stop anaylsing"
+                self.delete_urls_if_domains(domain)
+              when "a"
+                white_list.add_urls_with_same_domain_as url, self
               when "d"
                 domain = disavowed.add_domain_from_url(url)
                 self.delete_url url
-                puts "Attempting to remove URLs with the domain #{domain}"
+                puts "Attempting to remove URLs with the domain #{domain} from imported links to stop anaylsing"
                 self.delete_urls_if_domains(domain)
               when "u"
                 disavowed.add_url(url)
@@ -54,18 +74,48 @@ module DisavowTool
     end
 
     :private
-    def display_menu
-      puts "[w] to send to whitelist [d] to send to Disavow as a domain [u] to send to Disavow as a URL [o] to open the URL."
+    def menu
+      message = ""
+      message = "[w] Whitelist url [W] Whitelist the entire domain [a] whitelist as url All urls with this domain\n" if OPTIONS.whitelist
+      message += "[d] Disavow as domain [u] Disavow as a URL [o] to open the URL."
     end
 
     def open_browser_option(input, link)
       if input == "o"
           if Gem.win_platform? then system "start chrome #{link}" else system "open -a safari #{link}" end
           puts "Opening #{link}...".blue
-          display_menu
+          puts menu
           return true
         end
     end
 
+    def website_title(url)
+      begin
+        Timeout::timeout(SECONDS_TITTLE_REQUEST) do
+          page = Nokogiri::HTML(open(URI.escape(url)))
+          return "Empty Title" if page.css("title").blank?
+          return page.css("title")[0].text
+        end
+      rescue Timeout::Error => e
+        return "Empty Title — Request Time Out: #{e}"
+      rescue OpenURI::HTTPError => e
+        return "Empty Title. HTTP Error: #{e}"
+      rescue SocketError => e
+        return "Empty Title. Can't open site: #{e}"
+      rescue
+        return "Empty Tittle "
+      end
+
+    end
+
+    def urls_with_same_domain(url)
+      domain = URI.parse(URI.escape(url)).host
+      counter = 0
+      self.each do |link|
+        counter += 1 if URI.parse(URI.escape(link)).host == domain
+      end
+      counter
+    end
+
   end
 end

data/lib/disavow_tool/list.rb

@@ -20,6 +20,10 @@ module DisavowTool
         puts "Importing file: #{file}"
         File.readlines(file).each do |line|
           line.chomp!
+          if comment?(line) || line.blank?
+            puts "cleaning comment or empty line: #{line}" if @verbose_hard
+            next
+          end
           unmodified_line = line
           clean_line!(line)
           puts import_message(unmodified_line).light_blue if @verbose_hard
@@ -42,8 +46,7 @@ module DisavowTool
     def add_url(url, list=nil)
       list = list || @list
       if list.add? url
-        color_url = url.on_yellow
-        puts add_url_message(color_url).blue if @verbose
+        puts add_url_message(url).blue if @verbose
       else
         puts "Not adding #{url}. Already in the list.".red if @verbose
       end
@@ -59,7 +62,7 @@ module DisavowTool
     end
 
     def mass_delete_urls(urls_to_delete)
-      puts mass_delete_message if @verbose
+      puts mass_delete_message if @verbose_hard
       urls_to_delete.each do |link|
         color_link = link.on_yellow
         puts delete_url_message(color_link).red if @verbose
@@ -118,6 +121,10 @@ module DisavowTool
       end
     end
 
+    def comment?(line)
+      line.match(/^#/) ? true : false
+    end
+
     def each
       @list.each do |element|
         yield(element)