doubletake 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZTg2Nzg5ZDExOGQzN2QwNjdiNGMyODNmNTAwYTRhMGU4MGQzNDA1Zg==
+  data.tar.gz: !binary |-
+    OTM4MmRmM2FhOGExMGI4MzM4OWYxYWMwYzMzYWFmMTczNDdjZWM5OA==
+SHA512:
+  metadata.gz: !binary |-
+    NTM2Y2M2ZGVmNmIwMzM4NTk1MjFjZWNkYmNmNTY0MDg3NDg2NWZhNGJhZmRm
+    YmY5Mjk3ODRhYjEwNTY0YzE5ODA5ZDg1MWU3ZTI0OTBjNTU1ZjIxOGI3MzQy
+    ODJkN2YyNjlmZGM1MDRmYzFkNGQ2NDJmZGNhZTYyMGNjYWU0ZDM=
+  data.tar.gz: !binary |-
+    OWE4Y2UxMmYyMDk4N2M1NGZkM2UxNzE3Zjk5MDgxZTg3OTU5MGZhZThiYjUx
+    ZDllNTIzM2VmYmU3MTYxODc2M2VlZGE5NmI4NWZhNTdkZTlhMjNkNDIyNDhh
+    OWU4YzgyZTBmZmRmZDgzZWMwODk5OTM3NWJlYTM2ZjFkNjA2YTU=

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+./data/*
+.travis.yml

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in doubletake.gemspec
+gemspec

data/README.md

@@ -0,0 +1,147 @@
+# DoubleTake
+
+DoubleTake is a Visual regression testing tool for web-applications. It is packaged and distributed as a Ruby Gem.
+
+
+## Installation
+
+
+Install it using:
+
+    $ gem install doubletake
+
+## Usage
+
+To list all available features and functionality:
+    $ doubletake help
+
+To see details on usage of each feature:
+    $ doubletake help [COMMAND]
+
+## Quick Start Guide
+
+### Compare
+
+#### Why?
+
+CSS allows changes to be made easily but very difficult to make sure those changes don't produce unexpected
+consequences elsewhere in the project.
+
+#### How?
+
+DoubleTake is a command line tool and is configured using a YAML file.
+
+Generating and editing a config file is easy.
+```
+Run:
+   $ doubletake generate --file /tmp/mysite_config.yml
+```
+
+Use your preferred file editor and edit the config file.
+    ```
+    $ vim /tmp/mysite_config.yml
+    stage: 'https://mysite-stage-env.com'
+    prod:  'https://mysite-prod-env.com'
+    ```
+*NOTE*: Always add the scheme of the site while configuring i.e, http:// or https://
+
+Edit the ignored list to skip URIs or file formats. DoubleTake generates sensible defaults that can be extended.
+
+The window resolution can also be changed. The default desktop resolution is set to 1400x800
+
+DoubleTake is cable to authenticate with user credentials provided in the config file. This allows more in-depth scans.
+
+To do this simple make the following changes to the YAML file.
+```
+LOGIN: true
+LOGIN_URI: user/login
+USER_DOM_ID: edit-name
+USER_VALUE: username
+PASS_DOM_ID: edit-pass
+PASS_VALUE: secret_password
+```
+LOGIN_URI is the path where a user has to navigate to reach the login page. For example:
+https://mysite.com/login is the complete URL then the LOGIN_URI would be.
+```
+LOGIN_URI: login
+```
+USER_DOM_ID & PASS_DOM_ID are the web element ID which Selenium uses to find and type in the credentials.
+USER_VALUE & PASS_VALUE are the actual username and password to login.
+
+If you want DoubleTake to assert if it was successfully able to login with the given details:
+```
+LOGIN_CONFIRM: true
+LOGIN_CONFIRM_CHECK: homepage-onsite-team
+```
+
+LOGIN_CONFIRM_CHECK: is the ID of the web element it should find if it was successfully able to login.
+
+If you don't want to use this feature you can turn it off by setting
+ ```
+ LOGIN_CONFIRM: false
+ ```
+There are other values in the config file such as
+```
+bad_links: []
+to_be_scraped: []
+scraped: []
+LOGGED_IN: false
+```
+These are auto populated by DoubleTake during runtime and used with the Resume feature. Resume is covered later in this guide.
+
+Now that we have our config file ready we can launch our first scan.
+
+    $ doubletake compare --conf /tmp/mysite_config.yml
+
+If the Gem installed correctly and selenium was able to find the Firefox binary you will see two browsers loading pages and
+DoubleTake will be taking screenshots and does image comparision.
+
+Pages that have changed will have their screenshots saved in DoubleTake_data folder in your home directory.
+
+### Scrape
+
+#### Why?
+
+Short answer is: I have been asked several times in the past to make complete site backups and preserve how they render in
+multiple browsers. This was generally when a major revamp of the site was going to be made or as evidence to show to the client.
+There are also times when the UI/UX developers need to quickly see how all the pages look without having to browse every single
+page and prefer to see screenshots.
+
+#### How?
+
+All features in DoubleTake make use of the same config file format. We can reuse the config file we made earlier.
+
+Scrape only looks at the "stage: " value in the config and ignores  "prod: ".
+
+To scrape the URL:
+    $ doubletake scrape --conf /tmp/mysite_config.yml
+
+This time you should see only one Firefox browser launch and all the screenshots will be stored in DoubleTake_data folder in
+your home directory.
+
+### Resume
+
+#### Why?
+
+Cause shit happens and you don't want to restart a scan of a 4000 page e-commerce site after it has already scanned 2500 page.
+
+#### How?
+
+A progress config file is auto-generated when a Compare or Scrape scan is started. These files can be found in DoubleTake_data
+folder in the home directory
+
+Ex: ~/DoubleTake_data/desktop/progress_mysite.yml
+
+In addition to this progress config file we also need to known the type of the scan that is being resumed i.e, compare / scrape
+
+To Resume a scan:
+    $ doubletake resume --type compare --conf ~/DoubleTake_data/desktop/progress_mysite.com.yml
+
+
+## Contributing
+
+1. Fork it ( https://github.com/MelchiSalins/doubletake/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "doubletake"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/doubletake.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'doubletake/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "doubletake"
+  spec.version       = Doubletake::VERSION
+  spec.authors       = ["Melchi Salins"]
+  spec.email         = ["melchisalins@gmail.com"]
+
+  spec.summary       = %q{Visual regression testing tool and more }
+  spec.homepage      = "http://melchisalins.users.sf.net/"
+  spec.license       = "MIT"
+  spec.description   = "This is test description"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  end
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_runtime_dependency "pry", "~> 0.10.1"
+  spec.add_runtime_dependency 'rmagick', '>= 2.13.4', '~> 2.13.4'
+  spec.add_runtime_dependency 'selenium-webdriver', '>= 2.45.0', '~> 2.45.0'
+  spec.add_runtime_dependency 'thor', '~> 0.19.1'
+  spec.post_install_message = "Thanks for installing!"
+  spec.required_ruby_version = '>= 1.9.3'
+  spec.requirements << 'Firefox browser'
+end

data/exe/doubletake

@@ -0,0 +1,108 @@
+#!/usr/bin/env ruby
+
+require 'doubletake'
+require 'thor'
+require 'uri'
+require 'pry'
+
+class DoubleTake < Thor
+	include CrawlerHelper
+
+	desc "compare", "Starts comparing the stage and prod site configured in the YAML file"
+	option :conf, :desc => "--conf config_file.yml", :alias => "-c", :required => true
+	def compare
+		puts "* DoubleTake Compare Initializing..."
+		$config = YAML.load(File.open(options[:conf], "r"))
+		$config.LOGGED_IN = false # forcing this variable to false.
+		# puts $config.inspect
+		start
+	end # compare
+
+	desc "scrape ", "Scrape the specified Domain."
+	option :conf, :desc => "--conf config_file.yml",
+		   :alias => "-c", :required => true
+	def scrape
+		#TODO: This command doesn't support resume capability yet.
+		#	   Need to make resume a subcommand of comapre and scrape.
+		puts "* DoubleTake Scrape Initializing..."
+		$config = YAML.load(File.open(options[:conf], "r"))
+		$config.LOGGED_IN = false # forcing this variable to false.
+		# puts $config.inspect
+		start
+	end
+
+	desc "resume ", "Resumes a previous session of compare, Progress YAML file are located in the ./date directory."
+	option :conf, :required => true, :desc => "resume --conf progress_timestamp.yml",
+		   	 :alias => "-c"
+	option :type, :required => true, :desc => "Specify the type of scan being resumed (scrape/compare).",
+				 :alias => "-t"
+	def resume
+		puts "* Resuming previous session."
+		$config = YAML.load(File.open(options[:conf], "r"))
+		# puts $config.inspect
+		if ["scrape", "compare"].include? options[:type]
+			$cf = options[:type]
+		else
+			puts "--type should be either 'scrape' or 'compare'"
+			exit 1
+		end
+		start
+	end
+
+	desc "generate", "Generate a config file template."
+	option :file, :required => true, :desc => "Filename and path to write to. Ex: ~/config.yml"
+	def generate
+		begin
+			c = Configuration.new
+			File.open(options[:file],"w") {|f| f.write(c.to_yaml)}
+		rescue Exception => error
+			puts error
+			exit 1
+		end
+	end
+	no_commands do
+		def start
+			begin
+				$cf = caller[0][/`([^']*)'/, 1] unless $cf# Setting Calling function unless already set.
+				threads = []
+				$domains_list = {}
+				if $config.all_good?
+					puts "* $config.all_good? returned true."
+				else
+					puts "- There is error(s) in the suppplied config file. Cannot proceed."
+					exit(1)
+				end
+				$hostname = URI.parse($config.stage).host # This is required to name folders to store scan results.
+				puts $hostname
+				$domains_list[$hostname] = [$config.stage, $config.prod]
+
+				$domains_list.each do |site, urls|
+					# Here we pick each domain and spawn new threads
+					# for the regression tests.
+					# TODO: This is useful only when stage and prod are a list
+					#       of domains.
+					threads << Thread.new {task_1 = Crawler.new(site, urls[0], urls[1])
+					task_1.crawl if $config.LOGGED_IN == false
+					# Add code to check if creds are provided and then execute following:
+					if $config.LOGIN
+						task_1.login_to_as(urls[0], task_1.driver1)
+						task_1.login_to_as(urls[1], task_1.driver2) if $cf == "compare"
+						$config.to_be_scraped << task_1.driver1.current_url
+						task_1.crawl
+					end
+					task_1.clean_up
+					}
+				puts "* Thread Dispatched"
+				end
+				threads.each {|thr| thr.join}
+			rescue Interrupt
+				puts "\n* Exiting."
+			rescue Exception => e
+				puts e
+				puts "\n* Exiting."
+			end
+		end
+	end
+end
+
+DoubleTake.start(ARGV)

data/lib/config.rb

@@ -0,0 +1,76 @@
+require 'yaml'
+require 'pry'
+require './lib/crawler_lib.rb'
+
+class Configuration
+	attr_accessor :stage, :prod, :ignored, :DESKTOP, :to_be_scraped
+	attr_accessor :scraped, :bad_links, :LOGGED_IN
+	attr_reader :LOGIN_URI, :USER_VALUE, :USER_DOM_ID
+	attr_reader :PASS_VALUE, :PASS_DOM_ID
+	attr_reader :LOGIN_CONFIRM_CHECK, :LOGIN_CONFIRM, :LOGIN
+	attr_reader :URI_THRESHOLD, :IMAGE_THRESHOLD
+
+	def initialize
+		@stage = ""
+		@prod  = ""
+		@ignored = ["ignore_me", "not_important_url_prefix",".css", ".pdf", ".js", ".jpg", ".png", "video/pop", "user/logout", "?", "=", "#"]
+		@SCREEN_RESOLUTION = {:desktop => [1400,800]}
+		@IMAGE_THRESHOLD = 0
+		@LOGIN = true
+		@LOGIN_URI = 'user/login' # http://example.com/login
+		@USER_DOM_ID = 'edit-name'
+		@USER_VALUE = 'melchisalins'
+		@PASS_DOM_ID = 'edit-pass'
+		@PASS_VALUE = 'secret_password'
+		@LOGIN_CONFIRM = false
+		@LOGIN_CONFIRM_CHECK = 'homepage-onsite-team'
+		@bad_links = []
+		@to_be_scraped = []
+		@scraped = []
+		@LOGGED_IN = false
+	end
+
+	def all_good?
+		begin
+			# Fixes scheme of the URL if not present. This is needed by Selenium
+			return_value = false
+			if @stage.length <= 0 && @prod.length <= 0
+				puts "Stage and Production URL missing."
+				return_value = false
+				return return_value
+			else
+				@stage = fix_scheme(@stage) if URI.parse(@stage).scheme == nil
+				@prod  = fix_scheme(@prod)  if URI.parse(@prod).scheme == nil
+				return_value = true
+			end
+
+			if @LOGIN && @LOGIN_URI.nil? == false && @USER_DOM_ID.nil? == false && @USER_VALUE.nil? == false && @PASS_DOM_ID.nil? == false && @PASS_VALUE.nil? == false
+				return_value = true
+			else
+				puts "Please configure LOGIN parameters"
+				return_value = false
+				return return_value
+			end
+
+			if @LOGIN_CONFIRM && @LOGIN_CONFIRM_CHECK
+				return_value = true
+			elsif @LOGIN_CONFIRM == false
+				return_value = true
+			else
+				puts "Please configure LOGIN_CONFIRM_CHECK value"
+				return_value = false
+				return return_value
+			end
+
+			return return_value
+		rescue Exception => e
+			puts e
+			return false
+		end
+	end
+end
+# c = Configuration.new
+# File.open("test_yaml.yml","w") {|f| f.write(c.to_yaml)}
+# y = YAML.load(File.open("test_yaml.yml", "r"))
+# puts y.inspect
+# y.validate

data/lib/crawler_lib.rb

@@ -0,0 +1,66 @@
+require './lib/config.rb'
+require 'uri'
+require 'selenium-webdriver'
+
+module CrawlerHelper
+	def test
+		puts "Test call"
+	end
+
+	def fix_scheme(url)
+		puts "- No scheme provided for #{url}, trying to fix it."
+		driver = Selenium::WebDriver.for :firefox
+		driver.get("http://"+url) #assumes redirect to https is setup if it exists.
+		url_tmp = driver.current_url
+		scheme = URI.parse(url_tmp).scheme
+		driver.quit
+		puts "scheme is: #{scheme}"
+		return scheme+"://"+url
+	end
+
+
+	def sanitize(link)
+		# puts link
+		name = link.gsub(":", "")
+		name = name.gsub("/", "")
+		name = name.gsub("%", "")
+		name = name.gsub('\\', "")
+		name = name.gsub('.', "")
+		# puts name
+		return name
+	end
+	#
+	# def bad_link?(each_link)
+	# 	# return True if these characters exists in
+	# 	# the URL: $, #, png, css, js, jpg, pdf
+	# 	# Check for both upper and lower case ^
+	# 	begin
+	# 		each = each_link.upcase
+	# 		if each.include?("?") || each.include?("#") || each.include?(".PNG") || each.include?(".CSS") || each.include?("JS") || each.include?("JPG") || each.include?("PDF") || each.include?("/VIDEO/POP")
+	# 				# puts "Bad Link: " + each.to_s
+	# 				return true
+	# 		else
+	# 			return false
+	# 		end
+	# 	rescue Exception => e
+	# 		puts e.message
+  # 			puts e.backtrace.inspect
+  # 		end
+	#
+	# end
+
+	def do_not_ignore?(each_link, scraped)
+		# This checks if the passed link should be
+		# scraped or not based on:
+		# Has it already been scraped, is it bad_link?
+		# puts each_link
+		# puts scraped.class
+		if scraped.include?(each_link)
+			return false
+		elsif bad_link?(each_link)
+			return false
+		else
+			return true
+		end
+	end
+end

data/lib/doubletake.rb

@@ -0,0 +1,8 @@
+require "doubletake/version"
+require './lib/crawler_lib.rb'
+require './lib/config.rb'
+require './lib/site_context.rb'
+
+module Doubletake
+  # Your code goes here...
+end

data/lib/doubletake/version.rb

@@ -0,0 +1,3 @@
+module Doubletake
+  VERSION = "0.1.0"
+end

data/lib/site_context.rb

@@ -0,0 +1,248 @@
+require 'fileutils'
+require 'selenium-webdriver'
+require './lib/crawler_lib.rb'
+require './lib/config.rb'
+require 'pry'
+require 'RMagick'
+require 'yaml'
+require 'csv'
+
+# require 'pry-debugger'
+
+# class Configuration
+# 	# attr_accessor :stage, :prod, :ignored, :DESKTOP
+# 	def initialize
+# 		@stage = "https://rialto-stage.equiem.com.au"
+# 		@prod  = "https://atrialto.com"
+# 		@ignored = ["ignore_me", "not_important_url_prefix",".css", ".pdf", ".js", ".jpg", ".png", "video/pop", "user/logout", "?", "=", "#"]
+# 		@SCREEN_RESOLUTION = {:desktop => [1400,800], :mobile => [300,150]}
+# 		@IMAGE_THRESHOLD = 0
+# 		@LOGIN = true
+# 		@LOGIN_URI = 'login' # http://example.com/login
+# 		@USER_DOM_ID = 'edit-name'
+# 		@USER_VALUE = 'melchisalins'
+# 		@PASS_DOM_ID = 'edit-pass'
+# 		@PASS_VALUE = 'secret_password'
+# 		@LOGIN_CONFIRM = true
+# 		@LOGIN_CONFIRM_CHECK = 'homepage-onsite-team'
+# 	end
+# end
+
+class SiteContext
+	attr_accessor :driver
+
+	def initialize
+		puts "SiteContext initializes!"
+	end
+
+	def set_driver(browser = :chrome, remote = "http://192.168.15.43:4444/wd/hub/")
+		# driver = Selenium::WebDriver.for(:remote, :url => remote, :desired_capabilities => browser)
+		driver = Selenium::WebDriver.for :firefox
+		return driver
+	end
+
+	def login_to_as(site, driver)
+		$config.LOGGED_IN = true
+		driver.get(site + $config.LOGIN_URI)
+		username = driver.find_element(:id, $config.USER_DOM_ID)
+		username.clear
+		username.send_keys($config.USER_VALUE)
+		password = driver.find_element(:id, $config.PASS_DOM_ID)
+		password.clear
+		password.send_keys($config.PASS_VALUE+"\n")
+
+		if "Terms and Conditions".include?(driver.title)
+			driver.find_element(:id, "edit-legal-accept").click
+			driver.find_element(:id, "edit-save").click
+		end
+
+		if driver.find_element(:class, $config.LOGIN_CONFIRM_CHECK) != nil
+			return true
+		else
+			return false
+		end
+	end #End of Method login_to_as
+end
+
+class Crawler < SiteContext
+	include CrawlerHelper
+	include Magick
+	attr_accessor :site, :driver1, :driver2, :progress
+
+	class Progress
+		attr_accessor :driver1_type, :driver2_type
+		attr_accessor :stage, :prod
+		attr_accessor :bad_links, :to_be_scraped, :scraped
+
+		def initialize
+			@driver1_type = ""
+			@driver2_type = ""
+			@stage = ""
+			@prod  = ""
+			@bad_links    = []
+			@to_be_scraped = []
+			@scraped       = []
+		end
+	end
+
+
+	def initialize(site, test, base, browser = :firefox)
+		@site = site.to_s
+		FileUtils::mkdir_p "#{ENV['HOME']}/DoubleTake_data/desktop/"+@site
+		puts "* Screenshots are saved in #{ENV['HOME']}/DoubleTake_data/desktop/"+@site
+		$config.to_be_scraped << test
+		@test_domain_length = test.length
+		puts "Crawler initialized"
+		@driver1 = SiteContext.new
+		@driver1 = @driver1.set_driver(browser)
+		@driver1.get(test)
+		@driver1.manage.window.resize_to(1400, 800) #Desktop size
+		unless $cf == "scrape"
+			@driver2 = SiteContext.new
+			@driver2 = @driver2.set_driver(browser)
+			@driver2.get(base)
+			@driver2.manage.window.resize_to(1400, 800) #Desktop size
+		end
+	end
+
+	def clean_up
+		@driver1.quit
+		@driver2.quit unless $cf == "scrape"
+		puts "Destroyed WebDriver instances."
+	end
+
+	def bad_link?(link)
+		# This should populate bad_links[] and return bool
+		# regarding the link being passed in.
+		# $config.scraped << link
+		# Bad link could be parameterised URLs(?, #) or
+		# External domains such as facebook, twitter etc. or
+		# link is non http Ex: mailto: ftp: file: etc.
+		# puts $config.scraped
+		if link.include?("$") || link.include?("#") || link.include?(".png") || link.include?(".js")
+			puts "Bad Link: "+link
+			$config.scraped << link if ($config.scraped.include?(link) == false)
+			$config.bad_links << link if ($config.bad_links.include?(link) == false)
+			return true
+		elsif link.include?(".pdf") || link.include?(".jpeg") || link.include?(".css") || link.include?(".jpg")
+			puts "Bad Link: "+link
+			$config.scraped << link if ($config.scraped.include?(link) == false)
+			$config.bad_links << link if ($config.bad_links.include?(link) == false)
+			return true
+		elsif link.include?("video/pop") || link.include?("?") || link.include?("/user/logout")
+			puts "Bad Link: "+link
+			$config.scraped << link if ($config.scraped.include?(link) == false)
+			$config.bad_links << link if ($config.bad_links.include?(link) == false)
+			return true
+		elsif link[0..3] != "http" #TODO: This doesn't seem to work.
+			puts "Bad Link: "+link
+			$config.scraped << link if ($config.scraped.include?(link) == false)
+			$config.bad_links << link if ($config.bad_links.include?(link) == false)
+			return true
+		elsif link[0..@test_domain_length-1] != $config.stage || link.include?("%")
+			puts "Out of Scope: "+link
+			$config.scraped << link if ($config.scraped.include?(link) == false)
+			$config.bad_links << link if ($config.bad_links.include?(link) == false)
+			return true
+		elsif $config.scraped.include?(link)
+			puts "Already scraped: "+link
+			$config.bad_links << link if ($config.bad_links.include?(link) == false)
+			return true
+		else
+			puts "Good Link: "+link
+			return false
+		end #End of `if`
+
+	end
+
+	def crawl
+		# unless $config.to_be_scraped.empty?
+		loop do
+			puts "length of	progress.to_be_scraped: #{$config.to_be_scraped.length.to_s}"
+			break if $config.to_be_scraped.length < 1
+			puts "length of @to_be_scrapped: #{$config.to_be_scraped.length.to_s}"
+			$config.to_be_scraped.each do |each_link|
+				puts "*  length of @to_be_scrapped: #{$config.to_be_scraped.length.to_s}"
+				puts "** length of $config.scraped: #{$config.scraped.length.to_s}"
+				puts "Working on: #{each_link}"
+				begin
+					@driver1.get(each_link)
+					#### This Code block collects New Links and cleans
+					#   	$config.to_be_scraped Array.
+					all_a_objs = @driver1.find_elements(:xpath, '//a')
+					all_a_objs.each do |each_a_obj|
+						if each_a_obj.attribute("href") != nil #Why? Cause some link obj are dicks and don't have a href
+							# TODO: ^^ This if should be changed to begin - rescue
+							$config.to_be_scraped << each_a_obj.attribute("href") if (each_a_obj.attribute("href").include?("http") && bad_link?(each_a_obj.attribute("href")) == false)
+						end
+					end #all_a_objs.each do |each_a_obj|
+					$config.to_be_scraped.uniq! # Remove duplicate links.
+					$config.to_be_scraped.each do |each_new_link|
+						#This code block cleans the	$config.to_be_scraped Array
+						$config.to_be_scraped =	$config.to_be_scraped - [each_new_link] if ($config.scraped.include?(each_new_link) || bad_link?(each_new_link))
+					end 	#$config.to_be_scraped.each do |each_new_link|
+					if $config.scraped.include?(each_link)
+						# In case a bad link makes it into the loop
+						# This code-block will skip over it.
+						# It will also delte it from the	$config.to_be_scraped Array
+						$config.to_be_scraped =	$config.to_be_scraped - [each_link]
+						puts "Already Scrapped linked creeped in: #{each_link}"
+						# next
+					end
+
+					#
+					### End of Code Block to collect URL's to be scraped
+					stage_uri = each_link[@test_domain_length..-1]
+					prod_link = $config.prod + stage_uri
+					# *****************************************
+					if $cf == "scrape"
+						warning_log = CSV.open("#{ENV['HOME']}/DoubleTake_data/desktop/#{@site}.csv", "a")
+						name = sanitize(stage_uri)
+						@driver1.save_screenshot("#{ENV['HOME']}/DoubleTake_data/desktop/"+@site+"/"+name+"_stage.png")
+						@driver1.find_elements(:css, ".messages.error").each do |ele|
+							warning_log << [@driver1.current_url, ele.text] unless ele.nil?
+							end
+						warning_log.close
+					else
+						@driver2.get(prod_link)
+						image_stuff(stage_uri)
+					end
+					# *****************************************
+					$config.scraped << each_link # Last Step: Marking the URL as scraped!
+					$config.scraped.uniq! # bad_link? may add duplicate entries
+					$config.to_be_scraped =	$config.to_be_scraped - [nil] # This was issue when .delete() was used which resulted in element replaced by nil
+					$config.to_be_scraped =	$config.to_be_scraped - [each_link]
+					$config.to_be_scraped.uniq!
+					File.open("#{ENV['HOME']}/DoubleTake_data/desktop/progress_#{@site}.yml", "w") {|f| f.write($config.to_yaml)}
+				rescue Selenium::WebDriver::Error::StaleElementReferenceError => e
+					puts "Stale element error occured moving to next link: #{stage_uri}"
+					puts e
+					next
+				rescue Exception => e
+					puts "Generic Exception occured"
+					binding.pry
+					puts e.backtrace
+					next
+				end #End of begin
+			end 	#$config.to_be_scraped.each do |each_link|
+		end # Loop do
+		puts "to_be_scraped: " + $config.to_be_scraped.to_s
+		puts "scraped      : " + $config.scraped.to_s
+	end # Crawl Ending
+
+	def image_stuff(stage_uri)
+		name = sanitize(stage_uri)
+		@driver1.save_screenshot("#{ENV['HOME']}/DoubleTake_data/desktop/"+@site+"/"+name+"_stage.png")
+		@driver2.save_screenshot("#{ENV['HOME']}/DoubleTake_data/desktop/"+@site+"/"+name+"_prod.png")
+		# a, b = IO.read("#{ENV['HOME']}/DoubleTake_data/desktop/stage_"+@site+"/"+name+".png")[0x10..0x18].unpack('NN')
+		img1 = ImageList.new("#{ENV['HOME']}/DoubleTake_data/desktop/"+@site+"/"+name+"_stage.png")
+		img2 = ImageList.new("#{ENV['HOME']}/DoubleTake_data/desktop/"+@site+"/"+name+"_prod.png")
+		diff_img, diff_metric  = img1[0].compare_channel( img2[0], Magick::MeanSquaredErrorMetric )
+		if diff_metric > $config.IMAGE_THRESHOLD
+			diff_img.write("#{ENV['HOME']}/DoubleTake_data/desktop/"+@site+"/"+name+"_diff.png")
+		else
+			File.delete("#{ENV['HOME']}/DoubleTake_data/desktop/"+@site+"/"+name+"_stage.png")
+			File.delete("#{ENV['HOME']}/DoubleTake_data/desktop/"+@site+"/"+name+"_prod.png")
+		end # if diff_metric > $IMAGE_THRESHOLD
+	end #def image_stuff(image1, image2)
+end #Class Crawler < SiteContext

metadata

@@ -0,0 +1,158 @@
+--- !ruby/object:Gem::Specification
+name: doubletake
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Melchi Salins
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-04-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+- !ruby/object:Gem::Dependency
+  name: rmagick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.13.4
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.13.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.13.4
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.13.4
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.45.0
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.45.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.45.0
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.45.0
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+description: This is test description
+email:
+- melchisalins@gmail.com
+executables:
+- doubletake
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- doubletake.gemspec
+- exe/doubletake
+- lib/config.rb
+- lib/crawler_lib.rb
+- lib/doubletake.rb
+- lib/doubletake/version.rb
+- lib/site_context.rb
+homepage: http://melchisalins.users.sf.net/
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: Thanks for installing!
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- Firefox browser
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Visual regression testing tool and more
+test_files: []