cohesion 1.0.2 → 1.0.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YjRmZDJmNTNjNDJmYjRlODcxNDA1MWQ4YTgzYmNlOWM0NWEzMDRiMw==
+    MmZhYjJjNDYwY2FlYmU0MjljNDg0M2JkYWUyMmJhMWQ3MjQ3MzQ4Yw==
   data.tar.gz: !binary |-
-    YWM0M2I0NGNhNjQwMWVlYWU1ZGNlNmQ2MmE1N2NiYWE1OWY5YjYyYw==
+    ODRmNzUwNjRjMjAxZmQ0YmI4NTg1ODEyZmY2ODQ0NWI4YjQ3MWIyMw==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    NTZhODQ4ZDYyYzU2Yjg4ODZiZDUxMmZjYzliZjg1ZDk3YTlhOTg2ZGNlYzZm
-    ZDNkNzg1YmYxNzI3M2IzMGQ2OGZjMzgzNGVhMjNjYTQyMGNiMjc3YzQ3NmFk
-    OWFiMTI5YjhlMGM4YzVkMWJjMjZjZTNjZWMyMThlZjhiZjFiZjc=
+    NGU5Yjg3YjQ4ZmQyYmNlYzAwMDQ4MGY3NjYyZmY2ZDA1MTc5ZmU3M2ZmMTQ4
+    OGNiOTFmMmI0YjI0NDRmYmMwYzE3NzQ4NGMyNDAwODc4ODBmYWI4YjhhMTRl
+    YWI1Y2ZmMjFmMzExOGNkNmQ5NWU2Yjk2YzliMjVjYjZmN2U4ZTc=
   data.tar.gz: !binary |-
-    ZmY0ZGFjODA4YzcyZmZkYzJiZDcxYWEwZmI2NzFhYzBjY2YyMTRjMjMzNmE0
-    NGJiZjkwMzE1NTBlZmVlZjI0ZDY0NjhkNDU5NzJlYjY3YmRhODY5ZjgyNWRi
-    MjVlNTQ1ZTg2Y2FkYzRmZGFjNzgyY2IwOTU0ZGEwYjYyYWE4YTU=
+    ZWZjMTU3YzRmMzc1M2YyNjA3OGNmZDhhNzk1ODhiNzE0N2IyZGM2ZThkMjIw
+    ZjZlM2EzZDE4Y2QwNmYwOTAzOGQxNGRjOTIwMDk2NGVjNzljZDQ3ZGI0YjZi
+    N2EyMDc5YWUwOTdjMGYyMjMyYjQ4NDg3YjhlMTA5MDVmNWYxYTA=

data/bin/cohesion

@@ -25,36 +25,40 @@ opts = Slop.parse(:help => true) do
   on 'output=', 'Path to output data to'
   on 'output_format=', "Output format, csv or json"
 
+  on 'c', 'clearcache', "Clear the cache"
   on 'v', 'verbose', 'Display crawl information'
   on 'd', 'debug', 'Display debug information'
   on 'w', 'web_statistics', 'Start web stats server'
 end
 
+if opts[:clearcache]
+  Cohesion::Cache.clear
+end
 if opts[:url]
 
   options = opts.to_hash.delete_if { |k, v| v.nil? || k == :url}
 
+  options[:output_format] = "json" unless options.has_key? :output_format
   options[:seed_urls] = File.open(options[:seed_urls][0]).read.split("\n") if options[:seed_urls] && File.exists?(options[:seed_urls][0])
   options[:internal_urls] = File.open(options[:internal_urls][0]).read.split("\n") if options[:internal_urls] && File.exists?(options[:internal_urls][0])
   options[:external_urls] = File.open(options[:external_urls][0]).read.split("\n") if options[:external_urls] && File.exists?(options[:external_urls][0])
 
   failures = Cohesion::Check.site(opts[:url], options)
-  if failures.count == 0
+  if failures[:missing].count == 0
     exit(true)
   else
-    if opts[:output]
+    if options[:output]
       output = []
-      failures.each do |failure|
+      failures[:missing].each do |failure|
         output << {:error_page => failure[:issue][:url], :inbound_links => failure[:inbound]}
       end
 
-      opts[:output_format] = "json" unless opts[:output_format]
-      if opts[:output_format] == "json"
-        File.open(opts[:output], 'w') do |f|
+      if options[:output_format] == "json"
+        File.open(options[:output], 'w') do |f|
           f.write output.to_json
         end
-      elsif opts[:output_format] == "csv"
-        CSV.open(opts[:output], "wb") do |csv|
+      elsif options[:output_format] == "csv"
+        CSV.open("missing_#{options[:output]}", "wb") do |csv|
           csv << ["404 Url", "Page that contains link"]
           output.each do |line|
             line[:inbound_links].each do |link|
@@ -62,10 +66,18 @@ if opts[:url]
             end
           end
         end
+        CSV.open("duplicate_#{options[:output]}", "wb") do |csv|
+          csv << ["Hash of Content", "Pages with duplicate content"]
+          failures[:duplicate].each do |md5, pages|
+            pages.each do |link|
+              csv << [md5, link]
+            end
+          end
+        end
       end
     end
     exit(false)
   end
 else
-  puts
+  puts opts
 end

data/lib/cohesion.rb

@@ -4,145 +4,10 @@ require 'cobweb'
 require 'ptools'
 require 'digest/md5'
 
+require 'cohesion/check'
+require 'cohesion/cache'
 require 'cohesion/railtie' if defined?(Rails)
 
 module Cohesion
-  class Check
 
-    def self.rails_text
-      puts "WARNING - not working yet..."
-      root_path = Rails.root.to_s
-      Dir.glob("**/*").each do |filename|
-        unless File.directory?(filename) || File.binary?(filename) || filename.ends_with?(".rdb")
-          f = File.open(filename, "r")
-          content = f.read()
-          f.close
-          if content =~ /(https?:\/\/[a-zA-Z0-9\.\/\-_%&\?]+)/
-            print "Checking #{$1} "
-            begin
-              status_code = Cobweb.new(:raise_exceptions => true).head($1)[:status_code].to_i
-              if status_code != 200
-                puts " [#{status_code}] \e[31m\u2717\e[0m"
-              else
-                puts "\e[32m\u2713\e[0m"
-              end
-            rescue SocketError
-              status_code = 0
-              puts " [DNS Failed] \e[31m\u2717\e[0m"
-            end
-          end
-        end
-      end
-    end
-
-    def self.rails_object
-      puts "WARNING - not working yet..."
-      root_path = Rails.root.to_s
-      #app_name = Rails.application.name
-      #puts "Checking #{app_name}..."
-      app = CobwebSample::Application
-      app.routes.default_url_options = { :host => 'xxx.com' }
-
-      Dir.glob("app/controllers/**/*").each do |filename|
-        controller_name = filename.gsub(".rb","").split("/")[-1].classify
-        unless controller_name == "ApplicationController"
-          puts "Processing #{controller_name}"
-          controller = controller_name.constantize.new
-
-          view = ActionView::Base.new(ActionController::Base.view_paths, {}, controller)
-
-          view.view_paths = ActionController::Base.view_paths
-          view.extend ApplicationHelper
-          view.controller = controller
-          view.class_eval do
-            include ApplicationHelper
-            include app.routes.url_helpers
-          end
-          begin
-            puts view.render(:template => '/tests/index.html.erb')
-          rescue => e
-            puts "Error rendering view: #{e.message}" 
-          end
-        end
-      end
-    end
-
-    def self.site(url, options={})
-      errors = []
-      failures = []
-
-      pages = {}
-
-      options[:cache] = options[:cache].to_i if options[:cache]
-      crawler_options = {:cache_type => :full, :crawl_linked_external => true, :store_inbound_links => true}.merge(options)
-
-      statistics = CobwebCrawler.new(crawler_options).crawl(url) do |page|
-        print page[:url]
-
-        duplicate = !pages[Digest::MD5.hexdigest(page[:body])].nil?
-        pages[Digest::MD5.hexdigest(page[:body])] = [] unless pages[Digest::MD5.hexdigest(page[:body])]
-        pages[Digest::MD5.hexdigest(page[:body])] << page[:url]
-
-        # if it was a 404 before, just check again not using the cache this time
-        if page[:status_code] == 404
-          page = Cobweb.new(crawler_options.merge(:cache => nil)).get(page[:url])
-        end
-
-        if page[:status_code] == 404 || duplicate
-          if duplicate
-            puts " [duplicate] \e[31m\u2717\e[0m"
-          else
-            puts " [#{page[:status_code]}] \e[31m\u2717\e[0m"
-          end
-          failures << page
-        else
-          puts " \e[32m\u2713\e[0m"
-        end
-      end
-
-      puts statistics.redis.namespace
-      puts statistics.get_statistics
-
-      total_inbound_failures = 0
-      total_failures = 0
-
-      issues = []
-      if failures.count == 0
-        puts "All links working!"
-      else
-        puts "Failed urls:"
-        failures.each do |f|
-          inbound_links = statistics.inbound_links_for(f[:url])
-          issues << {:issue => f, :inbound => inbound_links}
-
-          total_inbound_failures += inbound_links.count
-          total_failures += 1
-
-          puts ""
-          puts "#{f[:url]} [ #{f[:status_code]} ]"
-          inbound_links.each do |inbound_link|
-            puts "  - #{inbound_link}"
-          end
-        end
-
-        puts ""
-        puts "Duplicate Content"
-        puts ""
-        pages.select{|k,v| v.count > 1}.each do |k,v|
-          puts "Duplicate: #{k}"
-          v.map{|x| puts "  - #{x}" }
-        end
-
-
-        puts ""
-        puts "Total Failed URLs: #{total_failures}"
-        puts "Total Duplicates: #{pages.map{|d| d[1]}.select{|d| d.count > 1}.inject{|total, d| total + d.count}.count}"
-        puts "Total Inbound Failures (Pages linking to a 404): #{total_inbound_failures}"
-        puts ""
-      end
-      puts
-
-      return issues
-    end
-  end
 end

data/lib/cohesion/cache.rb

@@ -0,0 +1,15 @@
+module Cohesion
+  class Cache
+    def self.clear_public
+      Cobweb::Cache.flush_public
+    end
+    def self.clear_crawls
+      Cobweb::Cache.flush_all_private
+    end
+    def self.clear_crawl(crawl_id)
+      Cobweb::Cache.flush_crawl(crawl_id)
+    end
+
+  end
+
+end

data/lib/cohesion/check.rb

@@ -0,0 +1,138 @@
+module Cohesion
+  class Check
+
+    def self.rails_text
+      puts "WARNING - not working yet..."
+      root_path = Rails.root.to_s
+      Dir.glob("**/*").each do |filename|
+        unless File.directory?(filename) || File.binary?(filename) || filename.ends_with?(".rdb")
+          f = File.open(filename, "r")
+          content = f.read()
+          f.close
+          if content =~ /(https?:\/\/[a-zA-Z0-9\.\/\-_%&\?]+)/
+            print "Checking #{$1} "
+            begin
+              status_code = Cobweb.new(:raise_exceptions => true).head($1)[:status_code].to_i
+              if status_code != 200
+                puts " [#{status_code}] \e[31m\u2717\e[0m"
+              else
+                puts "\e[32m\u2713\e[0m"
+              end
+            rescue SocketError
+              status_code = 0
+              puts " [DNS Failed] \e[31m\u2717\e[0m"
+            end
+          end
+        end
+      end
+    end
+
+    def self.rails_object
+      puts "WARNING - not working yet..."
+      root_path = Rails.root.to_s
+      #app_name = Rails.application.name
+      #puts "Checking #{app_name}..."
+      app = CobwebSample::Application
+      app.routes.default_url_options = { :host => 'xxx.com' }
+
+      Dir.glob("app/controllers/**/*").each do |filename|
+        controller_name = filename.gsub(".rb","").split("/")[-1].classify
+        unless controller_name == "ApplicationController"
+          puts "Processing #{controller_name}"
+          controller = controller_name.constantize.new
+
+          view = ActionView::Base.new(ActionController::Base.view_paths, {}, controller)
+
+          view.view_paths = ActionController::Base.view_paths
+          view.extend ApplicationHelper
+          view.controller = controller
+          view.class_eval do
+            include ApplicationHelper
+            include app.routes.url_helpers
+          end
+          begin
+            puts view.render(:template => '/tests/index.html.erb')
+          rescue => e
+            puts "Error rendering view: #{e.message}" 
+          end
+        end
+      end
+    end
+
+    def self.site(url, options={})
+      errors = []
+      failures = []
+
+      pages = {}
+
+      options[:cache] = options[:cache].to_i if options[:cache]
+      crawler_options = {:cache_type => :full, :crawl_linked_external => true, :store_inbound_links => true}.merge(options)
+
+      statistics = CobwebCrawler.new(crawler_options).crawl(url) do |page|
+        print page[:url]
+
+        #duplicate = !pages[Digest::MD5.hexdigest(page[:body])].nil?
+        #pages[Digest::MD5.hexdigest(page[:body])] = [] unless pages[Digest::MD5.hexdigest(page[:body])]
+        #pages[Digest::MD5.hexdigest(page[:body])] << page[:url]
+
+        # if it was a 404 before, just check again not using the cache this time
+        if page[:status_code] == 404
+          page = Cobweb.new(crawler_options.merge(:cache => nil)).get(page[:url])
+        end
+
+        if page[:status_code] == 404 #|| duplicate
+          #if duplicate
+          #  puts " [duplicate] \e[31m\u2717\e[0m"
+          #else
+            puts " [#{page[:status_code]}] \e[31m\u2717\e[0m"
+          #end
+          failures << page.select{|k,v| [:url, :status_code].include?(k)}
+        else
+          puts " \e[32m\u2713\e[0m"
+        end
+      end
+
+      total_inbound_failures = 0
+      total_failures = 0
+
+      issues = {:missing => [], :duplicate => []}#pages.select{|k,v| v.count > 1}}
+      if failures.count > 0
+        puts "Failed urls:"
+        failures.each do |f|
+          inbound_links = statistics.inbound_links_for(f[:url])
+          issues[:missing] << {:issue => f, :inbound => inbound_links}
+
+          total_inbound_failures += inbound_links.count
+          total_failures += 1
+
+          puts ""
+          puts "#{f[:url]} [ #{f[:status_code]} ]"
+          inbound_links.each do |inbound_link|
+            puts "  - #{inbound_link}"
+          end
+        end
+      end
+
+      duplicate_page_count = 0#pages.map{|d| d[1].count}.select{|count| count > 1}.reduce(:+).to_i
+      if duplicate_page_count > 0
+
+        puts ""
+        puts "Duplicate Content"
+        puts ""
+        pages.select{|k,v| v.count > 1}.each do |k,v|
+          puts "Duplicate: #{k}"
+          v.map{|x| puts "  - #{x}" }
+        end
+
+      end
+
+      puts ""
+      puts "Total Failed URLs: #{total_failures}"
+      #puts "Total Duplicates: #{duplicate_page_count}"
+      puts "Total Inbound Failures (Pages linking to a 404): #{total_inbound_failures}"
+      puts ""
+
+      return issues
+    end
+  end
+end

data/lib/cohesion/version.rb

@@ -1,3 +1,3 @@
 module Cohesion
-  VERSION = "1.0.2"
+  VERSION = "1.0.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cohesion
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - Stewart McKee
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-23 00:00:00.000000000 Z
+date: 2013-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cobweb
@@ -59,6 +59,8 @@ files:
 - bin/cohesion
 - cohesion.gemspec
 - lib/cohesion.rb
+- lib/cohesion/cache.rb
+- lib/cohesion/check.rb
 - lib/cohesion/railtie.rb
 - lib/cohesion/version.rb
 - lib/tasks/cohesion.rake