cohesion 1.0.0 → 1.0.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NDQwNDM2N2Y4ZDczY2ZmYWRhMDliMTE0MDIxMDcxN2Y1ZDEzNDhlYQ==
+    YTAwMDdjZWJkOGNiMjZhZTU4YjNlYWE3Mzc1NGJlODFkNjc0MmM4Mw==
   data.tar.gz: !binary |-
-    OWM0ODU0OWNiNjFjODcwNGUwMTk4ZGIyY2ZhZWY0OTA0NWI1Y2FiNA==
+    OGYxMWU0YjdjYzc4ZmU3Y2YxNzg2NzJjYTRmN2UwZGIwMWFkYWNjMg==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    OGFlOGY1MDg0MjFlYmViMTMwNzZiMmM2NDIzMGY3MjEyMTFmZDkwZjA4NjVk
-    NmExNDc5ODhhNTVkZDBjYjZkOTgwOGJhMjlhY2M0YzMzMWE4OTk0ZTIxNDg2
-    YWVmMWQ1NTA1M2E2ODExMWFlZDNlN2MzODZlNzk1MTNkOTAzZTI=
+    NzkzYjI4MDY5ODYzZTBhOWNmNWVlODNlYjEwZmI2NTdkNTgzZGQ4MjA3M2Jl
+    NGQ0NzE5YTBhMGU1MjU4YmY3YmNhYzNhMDk2ZTNlZmIwYmI3MDAwYzM5NTQz
+    ZTJiYjA2MGQ5OTM5YmJiNjIyMmRjZWZlMTc1MjM4YmM0OGJjOWI=
   data.tar.gz: !binary |-
-    ODIyNzcxZjA1MTBiZDkwYjMwY2E1ZGZkZjVjN2RjNjRiNWIwZWE3NDkwZGI1
-    ODBlOGIxMjkyNWE3NzkwZWU5MzM3YWI5ZmE2ZjI4OTY1YzM3ZDcxMzNkMmQx
-    OGI2YzlhNTA5NzU5MTA0NDVhZTZmYTI3MzhiYTc2MTI5ZGVlMzc=
+    YzFiODMyMDRlMjMyNjAzNTlmYzVkNzMyYjQ4MTIwY2VkY2U0MzQ0Y2M1NmY1
+    MDYwOTdiYmNkOTkwY2EyNjk5NjEyOTZhODYwNGU4OGM3N2IwMTk5YTBhMTgw
+    MmY3OTczOTIyMjU2ZGM2ZmRlZDFmMDBkYTczYzE5OTU1ODI2YWY=

data/Gemfile

@@ -3,7 +3,7 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in cohesion.gemspec
 gemspec
 
-gem "cobweb", ">= 1.0.12"
+gem "cobweb", ">= 1.0.15"
 
 gem 'rspec'
 gem 'resque'
@@ -16,4 +16,4 @@ gem "utf8cleaner"
 
 group :test do
   gem 'coveralls', require: false
-end
+end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    cohesion (0.0.7)
+    cohesion (1.0.1)
       cobweb
       ptools
 
@@ -12,7 +12,7 @@ GEM
     awesome_print (1.1.0)
     celluloid (0.14.1)
       timers (>= 1.0.0)
-    cobweb (1.0.12)
+    cobweb (1.0.15)
       addressable
       awesome_print
       haml
@@ -101,7 +101,7 @@ PLATFORMS
 
 DEPENDENCIES
   awesome_print
-  cobweb (>= 1.0.12)
+  cobweb (>= 1.0.15)
   cohesion!
   coveralls
   ptools

data/README.textile

@@ -34,20 +34,20 @@ h4. Options
 Cohesion allows you to control the crawl through various command line options.  These are available through help (see below) and are listed here.
 
 
-  * --url <start_url>                               Url to start crawl from
-  * --internal_urls <url1,url2>                     List of url patterns to include
-  * --external_urls <exclude_url1, exclude_url2>    List of url patterns to exclude
-  * --seed_urls <seed_url1,seed_url2>               Seed urls
-  * --crawl_limit <number_of_urls>                  Limit the crawl to a number of urls
-  * --thread_count <number_of_threads>              Set the number of threads used
-  * --cache <number_of_seconds>                     Sets the timeout for the cache, leave blank for no cache
-  * --timeout <timeout_in_seconds>                  Sets the timeout for http requests
-  * --output <filename>                             Path to output data to
-  * --output_format <format_for_output_file>        Output format, csv or json
-
-  * -v, --verbose                                   Display crawl information
-  * -d, --debug                                     Display debug information
-  * -w, --web_statistics                            Start web stats server
+  * --url <start_url>                                           Url to start crawl from
+  * --internal_urls <url1,url2 or filename>                     List of url patterns to include (comma seperated list or filename with url per line)
+  * --external_urls <exclude_url1, exclude_url2 or filename>    List of url patterns to exclude (comma seperated list or filename with url per line)
+  * --seed_urls <seed_url1,seed_url2 or filename>               Seed urls (comma seperated list or filename with url per line)
+  * --crawl_limit <number_of_urls>                              Limit the crawl to a number of urls
+  * --thread_count <number_of_threads>                          Set the number of threads used
+  * --cache <number_of_seconds>                                 Sets the timeout for the cache, leave blank for no cache
+  * --timeout <timeout_in_seconds>                              Sets the timeout for http requests
+  * --output <filename>                                         Path to output data to
+  * --output_format <format_for_output_file>                    Output format, csv or json
+
+  * -v, --verbose                                               Display crawl information
+  * -d, --debug                                                 Display debug information
+  * -w, --web_statistics                                        Start web stats server (http://localhost:4567)
 
 
 h4. Help

data/bin/cohesion

@@ -31,6 +31,10 @@ if opts[:url]
 
   options = opts.to_hash.delete_if { |k, v| v.nil? || k == :url}
 
+  options[:seed_urls] = File.open(options[:seed_urls][0]).read.split("\n") if options[:seed_urls] && File.exists?(options[:seed_urls][0])
+  options[:internal_urls] = File.open(options[:internal_urls][0]).read.split("\n") if options[:internal_urls] && File.exists?(options[:internal_urls][0])
+  options[:external_urls] = File.open(options[:external_urls][0]).read.split("\n") if options[:external_urls] && File.exists?(options[:external_urls][0])
+
   failures = Cohesion::Check.site(opts[:url], options)
   if failures.count == 0
     exit(true)

data/lib/cohesion.rb

@@ -1,6 +1,8 @@
+require 'bundler/setup'
 require "cohesion/version"
 require 'cobweb'
 require 'ptools'
+require 'digest/md5'
 
 require 'cohesion/railtie' if defined?(Rails)
 
@@ -69,17 +71,29 @@ module Cohesion
       errors = []
       failures = []
 
+      pages = {}
+
       options[:cache] = options[:cache].to_i if options[:cache]
       crawler_options = {:cache_type => :full, :crawl_linked_external => true, :store_inbound_links => true}.merge(options)
-      puts crawler_options
 
       statistics = CobwebCrawler.new(crawler_options).crawl(url) do |page|
         print page[:url]
+
+        duplicate = !pages[Digest::MD5.hexdigest(page[:body])].nil?
+        pages[Digest::MD5.hexdigest(page[:body])] = [] unless pages[Digest::MD5.hexdigest(page[:body])]
+        pages[Digest::MD5.hexdigest(page[:body])] << page[:url]
+
+        # if it was a 404 before, just check again not using the cache this time
         if page[:status_code] == 404
           page = Cobweb.new(crawler_options.merge(:cache => nil)).get(page[:url])
         end
-        if page[:status_code] > 399
-          puts " [#{page[:status_code]}] \e[31m\u2717\e[0m"
+
+        if page[:status_code] == 404 || duplicate
+          if duplicate
+            puts " [duplicate] \e[31m\u2717\e[0m"
+          else
+            puts " [#{page[:status_code]}] \e[31m\u2717\e[0m"
+          end
           failures << page
         else
           puts " \e[32m\u2713\e[0m"
@@ -111,8 +125,18 @@ module Cohesion
           end
         end
 
+        puts ""
+        puts "Duplicate Content"
+        puts ""
+        pages.select{|k,v| v.count > 1}.each do |k,v|
+          puts "Duplicate: #{k}"
+          v.map{|x| puts "  - #{x}" }
+        end
+
+
         puts ""
         puts "Total Failed URLs: #{total_failures}"
+        puts "Total Duplicates: #{pages.map{|d| d[1]}.select{|d| d.count > 1}.inject{|total, d| total + d.count}.count}"
         puts "Total Inbound Failures (Pages linking to a 404): #{total_inbound_failures}"
         puts ""
       end

data/lib/cohesion/version.rb

@@ -1,3 +1,3 @@
 module Cohesion
-  VERSION = "1.0.0"
+  VERSION = "1.0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cohesion
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Stewart McKee
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-15 00:00:00.000000000 Z
+date: 2013-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cobweb