disavow_tool 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: af38dc15d1fd21dcab9a427645eff07ce7119266ea529d9237b9594c941e953b
+  data.tar.gz: ed632a61782d1e7748a8afc7b757c321a8c67cd8ecbe628954a2dc735badd041
+SHA512:
+  metadata.gz: 96595e35e141f4a54c291708e55af3dbfa938180a659db4e5f3b4a7e0202fd5552f3aa8bac2143330dbb5e366a5c33f487efe19d4f622cc4b3de6adab3b5ee6d
+  data.tar.gz: 2a3a9598ebacbd53201b754f2e03b1044ad02fc3095b2574dd80659f367f295f6d53f19a58b8c6bee8255ac091630518bd934f2373499fbd0e9262da0fb6e064

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+
+/exports/

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.3.4
+before_install: gem install bundler -v 2.0.2

data/Gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in disavow_tool.gemspec
+gemspec
+
+# Specify what Ruby version is needed
+ruby '2.6.0'

data/Gemfile.lock

@@ -0,0 +1,53 @@
+PATH
+  remote: .
+  specs:
+    disavow_tool (0.1.0)
+      activesupport
+      colorize
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.2.3)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    colorize (0.8.1)
+    concurrent-ruby (1.1.5)
+    diff-lcs (1.3)
+    i18n (1.6.0)
+      concurrent-ruby (~> 1.0)
+    minitest (5.11.3)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.2)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.4)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.2)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  disavow_tool!
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+RUBY VERSION
+   ruby 2.6.0p0
+
+BUNDLED WITH
+   2.0.2

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Guillermo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# DisavowTool
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/disavow_tool`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'disavow_tool'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install disavow_tool
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/disavow_tool.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "disavow_tool"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/disavow_tool.gemspec

@@ -0,0 +1,39 @@
+lib = File.expand_path("lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "disavow_tool/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "disavow_tool"
+  spec.version       = DisavowTool::VERSION
+  spec.authors       = ["Maesitos"]
+  spec.email         = ["noreply@example.com"]
+
+  spec.summary       = %q{ A simple tool to create Disavow tool }
+  spec.description   = %q{ A simple tool to create Disavow tool }
+  spec.homepage      = "https://github.com/maesitos/disavow_tool"
+  spec.license       = "MIT"
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/maesitos/disavow_tool"
+  spec.metadata["changelog_uri"] = "https://github.com/maesitos/disavow_tool"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # Development dependencies
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+
+  # Dependencies
+  spec.add_dependency "colorize"
+  spec.add_dependency "activesupport"
+end

data/exe/disavow_tool

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'disavow_tool'
+
+DisavowTool.run

data/lib/disavow_tool.rb

@@ -0,0 +1,157 @@
+#!/usr/bin/ruby
+require 'colorize'
+require 'io/console'
+require_relative 'disavow_tool/config'
+require_relative 'disavow_tool/version'
+require_relative 'disavow_tool/command_options.rb'
+require_relative 'disavow_tool/disavow_list.rb'
+require_relative 'disavow_tool/white_list.rb'
+require_relative 'disavow_tool/imported_links.rb'
+
+module DisavowTool
+
+  def self.run
+    puts "Importing new links".blue if OPTIONS.verbose
+    imported_links = ImportedLinks.new
+
+    puts "Importing Disavowed links".blue if OPTIONS.verbose
+    disavowed = DisavowList.new
+
+    if OPTIONS.whitelist
+      puts "Importing Whitelist links".blue if OPTIONS.verbose
+      white_list = WhiteList.new
+      puts "Cleagning imported links already in whitelist".blue if OPTIONS.verbose
+      imported_links.remove_known_links(white_list)
+    end
+
+
+    puts "Cleagning imported links already in Disavow".blue if OPTIONS.verbose
+    imported_links.remove_known_links(disavowed.links)
+
+    puts "Cleagning imported links with a domain existingin in Disavow".blue if OPTIONS.verbose
+    imported_links.remove_known_links_for_domain(disavowed.domains)
+
+    imported_links.summary
+
+    imported_links.analyse(disavowed, white_list)
+
+    disavowed.summary if OPTIONS.verbose
+    white_list.summary if OPTIONS.verbose
+
+    puts "Exporting...".red if OPTIONS.verbose
+    disavowed.export
+    white_list.export if OPTIONS.whitelist
+  end
+
+  # # We'll only analyse links that are not in
+  # # the Whitelist nor at Disavowed
+  # knwon_links = (@disavowed_links + @whitelisted_links).uniq
+  # @imported_links.each do |imported_link|
+  #   knwon_links.each do |knwon_link|
+  #     if imported_link == knwon_link
+  #       @imported_links.delete(knwon_link)
+  #       if Parameters::OPTIONS[:verbose]
+  #         puts "URL #{knwon_link} already in Disavowed or Whitelist".green
+  #         puts "  Remowing #{knwon_link.on_green} from imported links.".red
+  #       end
+  #     end
+  #   end
+  # end
+  #
+  # @imported_links.each do |imported_link|
+  #   domain2delete = @disavowed_domains.find do |disavowed_domain|
+  #     imported_link.match(/.+#{Regexp.escape(disavowed_domain)}.+/)
+  #   end
+  #
+  #   next if domain2delete.nil?
+  #   @disavowed_domains.delete(domain2delete) # Carefull
+  #   if Parameters::OPTIONS[:verbose]
+  #     puts "Found imported links with the already in Disavowed domain #{domain2delete}  ".green
+  #   end
+  #
+  #   @imported_links.reject! do |candidate_to_remove|
+  #     if candidate_to_remove.match(/.+#{Regexp.escape(domain2delete)}.+/)
+  #       puts "  Removing #{candidate_to_remove.on_yellow} from imported links.".red if Parameters::OPTIONS[:verbose]
+  #       true
+  #     end
+  #   end
+  # end
+  #
+  # if  Parameters::OPTIONS[:verbose]
+  #   puts "\n\n*********\n* SUMARY\n*********"
+  #   puts "Imported:"
+  #   puts " * #{DISAVOWED_LINKS.count} disavowed links"
+  #   puts " * #{DISAVOWED_DOMAINS.count} disavowed domains"
+  #   puts " * #{IMPORTED_LINKS.count} links to analyse"
+  #   puts " * #{WHITELISTED_LINKS.count} whitelisted links"
+  #   puts "Liks Discated: #{IMPORTED_LINKS.count - @imported_links.count}"
+  #   puts "Links Pending to be Verified: #{@imported_links.count}".green
+  # end
+  #
+  #
+  # #Reset vars
+  # @disavowed_links = DISAVOWED_LINKS.clone
+  # @disavowed_domains = DISAVOWED_DOMAINS.clone
+  # @whitelisted_links = WHITELISTED_LINKS.clone
+  #
+  # @imported_links.each do |link|
+  #   puts "\n\n#{"*"*100}\n* Analysing url: #{link.on_green}\n#{"*"*100}"
+  #   puts "[w] to send to whitelist [d] to send to Disavow as a domain [u] to send to Disavow as a URL [o] to open the URL."
+  #   input = $stdin.getch
+  #   if input == "o"
+  #     if Gem.win_platform? then system "start chrome #{link}" else system "open -a safari #{link}" end
+  #     puts "Opening #{link}...".blue
+  #     puts "Continue with options [w], [d] or [u]"
+  #     input = $stdin.getch
+  #   end
+  #   case input
+  #     when "w"
+  #       @whitelisted_links << link
+  #       puts "+++ Inserting #{link.on_green} in #{"whitelist".on_green}".light_blue if Parameters::OPTIONS[:verbose]
+  #
+  #     when "d"
+  #       host_to_disavow = URI.parse(link).host
+  #       puts "+++ Inserting #{host_to_disavow.on_yellow} in #{"disavowed Domains".on_yellow}".light_blue if Parameters::OPTIONS[:verbose]
+  #       @disavowed_domains << host_to_disavow
+  #
+  #       puts "---  Removing #{link} from pending to verify list.".red if Parameters::OPTIONS[:verbose]
+  #       @imported_links.delete link
+  #
+  #       puts "Looking for links with the domain #{host_to_disavow} in the links pending to verified...".blue
+  #       links_without_target_domain = @imported_links.reject{|imported_link| host_to_disavow == URI.parse(imported_link).host}
+  #       links_found = @imported_links.count - links_without_target_domain.count
+  #
+  #       if(links_found > 0)
+  #         puts "#{links_found} links pending to veryfy with the domain #{host_to_disavow}".green
+  #         (@imported_links - links_without_target_domain).each do |link_to_remove|
+  #           puts "--- Removing #{link_to_remove.on_yellow} from pending to verify list.".red if Parameters::OPTIONS[:verbose]
+  #           @imported_links.delete link_to_remove
+  #         end
+  #       else
+  #         puts "No links pending to verify with the domain #{host_to_disavow}".light_blue if Parameters::OPTIONS[:verbose]
+  #       end
+  #
+  #     when "u"
+  #       @disavowed_links << link
+  #       @imported_links.delete link
+  #       puts "+++ Inserting #{link.on_yellow} in #{"disavowed links".on_yellow}".light_blue if Parameters::OPTIONS[:verbose]
+  #     else raise "Invalid character."
+  #   end
+  # end
+  #
+  # if  Parameters::OPTIONS[:verbose]
+  #   puts "\n\n*********\n* SUMARY\n*********"
+  #   puts "Links added to whitelist:"
+  #   (@whitelisted_links - WHITELISTED_LINKS).each{ |whitelisted| puts "New whitelisted #{whitelisted}".green}
+  #
+  #   puts "Links added to disavow:"
+  #   (@disavowed_links - DISAVOWED_LINKS).each{ |disavowed_link| puts "New disavowed link #{disavowed_link}".red}
+  #
+  #   puts "Domain added to disavow:"
+  #   (@disavowed_domains - DISAVOWED_DOMAINS).each{ |disavowed_domain| puts "New disavowed link #{disavowed_domain}".red}
+  #
+  # end
+
+
+
+end

data/lib/disavow_tool/command_options.rb

@@ -0,0 +1,63 @@
+require 'optparse'
+require 'ostruct'
+
+module DisavowTool
+  class Commands
+    def self.parse_inputs
+       Commands.new.parse(ARGV)
+    end
+
+    def parse(args)
+      options = OpenStruct.new
+      options.library = []
+      options.whitelist  = false
+      options.verbose = false
+      options.hardcore_verbose = false
+
+      opt_parser = OptionParser.new do |opts|
+          opts.banner = "Usage: disavow.rb [options] --disavow FILE --import file_1,file_2,file_3 [--whitelist file1,file2,file3]"
+          opts.separator ""
+          opts.separator "Requited options:"
+          opts.on("-d","--disavow FILE", Array, "Disavow file as exported from Google Search Console") do |file|
+            options.disavow_file = file
+          end
+          opts.on("-i","--import file_1,file_2", Array, "List of URLS to analyse. The file must have one URL per line") do |file|
+            options.import_files = file
+          end
+
+          opts.separator ""
+          opts.separator "Optional options:"
+          opts.on("--whitelist file1,file2,file3", Array, "Enter any number of whitelisted files",
+                                                       "whitelisted files must have one URL per line") do |file|
+            options.whitelist  = true
+            options.whitelist_files  = file
+          end
+
+          opts.on("-v", "--verbose", "Vervose mode") do
+            options.verbose = true
+          end
+          opts.on("-V", "--hardcore-vervose", "Print out even your mama") do
+            options.hardcore_verbose = true
+            options.verbose = true            # Hardcose verbose includes regular verbose
+          end
+
+          opts.separator ""
+          opts.separator "Common options:"
+          # No argument, shows at tail.  This will print an options summary.
+          opts.on_tail("-h", "--help", "Show this message") do
+            puts opts
+            exit
+          end
+          opts.on_tail("--version", "Show version") do
+            puts VERSION
+            exit
+          end
+      end
+
+      opt_parser.parse!(args)
+      options
+    end
+  end
+
+  OPTIONS = Commands.parse_inputs
+end