cohesion 0.0.6 → 0.0.7

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MzAyNzU3ZmQ0MTZhMmUxYWQ4ZjkyYjViMzY1ZjZlYjc1ODBmYTdjMw==
+  data.tar.gz: !binary |-
+    NjgwZmU5Yzc3ZDBlNTFiZmJlZmFjNmY2NDlhNGJjOTIxZjU1MmEzNA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    ZTk5YWVjMDU0ODA0ZjVhNGVmMjczYjE4MzA5NGE4NDQ1NGEzNzNiOWFmMjQ2
+    YWUxZGNjMjA5Y2NlNDQ0MWU4MmExMDg5M2QxNTExN2YxNzQzZTIzNjhiOThk
+    ZjNiMzIyZGE0NjU2MzAyODg2OGVkYjZiZmE1ZWUyZWFiZDg2NGQ=
+  data.tar.gz: !binary |-
+    NWQwMjZjYjRlMjU1YjA5ZGQ3MGZkY2U3NzkxN2M2ZjczMjAyN2I2OWE3MzIy
+    OTc5MjU3ZTc5Y2IxODY5N2E5MTEzOThmYjYwOTc0ZWIxZjdhNzA2MWExZjdj
+    MmJmNDQwZTE4MDA3N2IxODIxY2M5NDQ5OTJiMGYwYWYyZjU1N2I=

data/.gitignore

@@ -14,3 +14,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+.DS_Store

data/.ruby-gemset

@@ -0,0 +1 @@
+cohesion

data/.ruby-version

@@ -0,0 +1 @@
+ruby-1.9.3-p392

data/Gemfile

@@ -3,10 +3,17 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in cohesion.gemspec
 gemspec
 
-gem "cobweb", ">= 1.0.10"
+gem "cobweb", ">= 1.0.12"
 
 gem 'rspec'
+gem 'resque'
 gem 'awesome_print'
 gem 'sqlite3'
 gem 'ptools'
-gem "slop"
+gem "slop"
+gem 'sidekiq'
+gem "utf8cleaner"
+
+group :test do
+  gem 'coveralls', require: false
+end

data/Gemfile.lock

@@ -1,16 +1,18 @@
 PATH
   remote: .
   specs:
-    cohesion (0.0.5)
+    cohesion (0.0.6)
       cobweb
       ptools
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.3.2)
+    addressable (2.3.5)
     awesome_print (1.1.0)
-    cobweb (1.0.10)
+    celluloid (0.14.1)
+      timers (>= 1.0.0)
+    cobweb (1.0.12)
       addressable
       awesome_print
       haml
@@ -18,51 +20,79 @@ GEM
       namespaced_redis
       nokogiri
       redis
-      resque
       rspec
       sinatra
+      slop
       thin
+    colorize (0.5.8)
+    connection_pool (1.1.0)
+    coveralls (0.6.7)
+      colorize
+      multi_json (~> 1.3)
+      rest-client
+      simplecov (>= 0.7)
+      thor
     daemons (1.1.9)
-    diff-lcs (1.1.3)
-    eventmachine (1.0.0)
-    haml (4.0.0)
+    diff-lcs (1.2.4)
+    eventmachine (1.0.3)
+    haml (4.0.3)
       tilt
-    json (1.7.7)
-    multi_json (1.6.0)
+    json (1.8.0)
+    mime-types (1.24)
+    mini_portile (0.5.1)
+    mono_logger (1.1.0)
+    multi_json (1.7.9)
     namespaced_redis (1.0.4)
       redis
-    nokogiri (1.5.6)
+    nokogiri (1.6.0)
+      mini_portile (~> 0.5.0)
     ptools (1.2.2)
     rack (1.5.2)
-    rack-protection (1.3.2)
+    rack-protection (1.5.0)
       rack
-    redis (3.0.2)
-    redis-namespace (1.2.1)
+    redis (3.0.4)
+    redis-namespace (1.3.1)
       redis (~> 3.0.0)
-    resque (1.23.0)
+    resque (1.24.1)
+      mono_logger (~> 1.0)
       multi_json (~> 1.0)
-      redis-namespace (~> 1.0)
+      redis-namespace (~> 1.2)
       sinatra (>= 0.9.2)
       vegas (~> 0.1.2)
-    rspec (2.12.0)
-      rspec-core (~> 2.12.0)
-      rspec-expectations (~> 2.12.0)
-      rspec-mocks (~> 2.12.0)
-    rspec-core (2.12.2)
-    rspec-expectations (2.12.1)
-      diff-lcs (~> 1.1.3)
-    rspec-mocks (2.12.2)
-    sinatra (1.3.4)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.5)
+    rspec-expectations (2.14.2)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.3)
+    sidekiq (2.13.1)
+      celluloid (>= 0.14.1)
+      connection_pool (>= 1.0.0)
+      json
+      redis (>= 3.0)
+      redis-namespace
+    simplecov (0.7.1)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.7.1)
+    simplecov-html (0.7.1)
+    sinatra (1.4.3)
       rack (~> 1.4)
-      rack-protection (~> 1.3)
-      tilt (~> 1.3, >= 1.3.3)
-    slop (3.4.3)
-    sqlite3 (1.3.6)
-    thin (1.5.0)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    slop (3.4.6)
+    sqlite3 (1.3.7)
+    thin (1.5.1)
       daemons (>= 1.0.9)
       eventmachine (>= 0.12.6)
       rack (>= 1.0.0)
-    tilt (1.3.3)
+    thor (0.18.1)
+    tilt (1.4.1)
+    timers (1.1.0)
+    utf8cleaner (0.0.1)
     vegas (0.1.11)
       rack (>= 1.0.0)
 
@@ -71,9 +101,13 @@ PLATFORMS
 
 DEPENDENCIES
   awesome_print
-  cobweb (>= 1.0.10)
+  cobweb (>= 1.0.12)
   cohesion!
+  coveralls
   ptools
+  resque
   rspec
+  sidekiq
   slop
   sqlite3
+  utf8cleaner

data/README.textile

@@ -1,6 +1,11 @@
 
 h1. Cohesion
 
+!https://badge.fury.io/rb/cohesion.png!:http://badge.fury.io/rb/cohesion
+!https://gemnasium.com/stewartmckee/coffee_table.png!
+!https://coveralls.io/repos/stewartmckee/cohesion/badge.png?branch=master(Coverage Status)!:https://coveralls.io/r/stewartmckee/cohesion
+
+
 h2. Intro
 
 Cohesion is a group of tasks that allow you to check link cohesion in a site.
@@ -35,15 +40,16 @@ Cohesion allows you to control the crawl through various command line options.
   * --seed_urls <seed_url1,seed_url2>               Seed urls
   * --crawl_limit <number_of_urls>                  Limit the crawl to a number of urls
   * --thread_count <number_of_threads>              Set the number of threads used
+  * --cache <number_of_seconds>                     Sets the timeout for the cache, leave blank for no cache
   * --timeout <timeout_in_seconds>                  Sets the timeout for http requests
   * --output <filename>                             Path to output data to
+  * --output_format <format_for_output_file>        Output format, csv or json
 
   * -v, --verbose                                   Display crawl information
   * -d, --debug                                     Display debug information
   * -w, --web_statistics                            Start web stats server
 
 
-
 h4. Help
 
 Help with available options is available with --help

data/bin/cohesion

@@ -5,6 +5,7 @@ $LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
 
 require 'cohesion'
 require 'slop'
+require 'csv'
 
 opts = Slop.parse(:help => true) do
   banner 'Usage: cohesion [options]'
@@ -16,8 +17,10 @@ opts = Slop.parse(:help => true) do
   on 'crawl_limit=', 'Limit the crawl to a number of urls', :as => Integer
   on 'thread_count=', "Set the number of threads used", :as => Integer
   on 'timeout=', "Sets the timeout for http requests", :as => Integer
+  on 'cache=', "Sets the timeout for the cache, leave blank for no cache"
 
   on 'output=', 'Path to output data to'
+  on 'output_format=', "Output format, csv or json"
 
   on 'v', 'verbose', 'Display crawl information'
   on 'd', 'debug', 'Display debug information'
@@ -27,15 +30,31 @@ end
 if opts[:url]
 
   options = opts.to_hash.delete_if { |k, v| v.nil? || k == :url}
-  ap options
 
   failures = Cohesion::Check.site(opts[:url], options)
   if failures.count == 0
     exit(true)
   else
     if opts[:output]
-      File.open(opts[:output], 'w') do |f|
-        f.write failures.to_json
+      output = []
+      failures.each do |failure|
+        output << {:error_page => failure[:issue][:url], :inbound_links => failure[:inbound]}
+      end
+
+      opts[:output_format] = "json" unless opts[:output_format]
+      if opts[:output_format] == "json"
+        File.open(opts[:output], 'w') do |f|
+          f.write output.to_json
+        end
+      elsif opts[:output_format] == "csv"
+        CSV.open(opts[:output], "wb") do |csv|
+          csv << ["404 Url", "Page that contains link"]
+          output.each do |line|
+            line[:inbound_links].each do |link|
+              csv << [line[:error_page], link]
+            end
+          end
+        end
       end
     end
     exit(false)

data/lib/cohesion.rb

@@ -1,5 +1,5 @@
 require "cohesion/version"
-require 'cobweb'
+require '../cobweb/lib/cobweb'
 require 'ptools'
 
 require 'cohesion/railtie' if defined?(Rails)
@@ -68,8 +68,16 @@ module Cohesion
     def self.site(url, options={})
       errors = []
       failures = []
-      statistics = CobwebCrawler.new({:cache => 3600, :cache_type => :full, :crawl_linked_external => true, :store_refered_url => true}.merge(options)).crawl(url) do |page|
+
+      options[:cache] = options[:cache].to_i if options[:cache]
+      crawler_options = {:cache_type => :full, :crawl_linked_external => true, :store_inbound_links => true}.merge(options)
+      puts crawler_options
+
+      statistics = CobwebCrawler.new(crawler_options).crawl(url) do |page|
         print page[:url]
+        if page[:status_code] == 404
+          page = Cobweb.new(crawler_options.merge(:cache => nil)).get(page[:url])
+        end
         if page[:status_code] > 399
           puts " [#{page[:status_code]}] \e[31m\u2717\e[0m"
           failures << page
@@ -78,6 +86,12 @@ module Cohesion
         end
       end
 
+      puts statistics.redis.namespace
+      puts statistics.get_statistics
+
+      total_inbound_failures = 0
+      total_failures = 0
+
       issues = []
       if failures.count == 0
         puts "All links working!"
@@ -86,12 +100,20 @@ module Cohesion
         failures.each do |f|
           inbound_links = statistics.inbound_links_for(f[:url])
           issues << {:issue => f, :inbound => inbound_links}
+
+          total_inbound_failures += inbound_links.count
+          total_failures += 1
+
           puts ""
           puts "#{f[:url]} [ #{f[:status_code]} ]"
           inbound_links.each do |inbound_link|
             puts "  - #{inbound_link}"
           end
         end
+
+        puts ""
+        puts "Total Failed URLs: #{total_failures}"
+        puts "Total Inbound Failures (Pages linking to a 404): #{total_inbound_failures}"
         puts ""
       end
       puts

data/lib/cohesion/version.rb

@@ -1,3 +1,3 @@
 module Cohesion
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/spec/lib/cohesion_spec.rb

@@ -13,6 +13,21 @@ describe Cohesion do
   context "site" do
     it "should crawl the site"
     it "should return error pages"
+    it "should exclude urls given as external"
+    it "should default to accept one level of external"
+    it "should not allow one level external for excluded urls"
+    it "should detect 400 level errors"
+    it "should detect 500 level errors"
+    it "should write valid json file"
+    it "should not write to file if not requested"
+    it "should start the web stats when requested"
+    it "should return data when --help requested"
+    it "should return help when invalid params are passed"
+    it "should return help when no params are passed"
+    it "should print out success links with tick"
+    it "should print out failed links with cross"
+    it "should print out report at end of crawl"
+    it "should default to full cache"
   end
 
   context "rails" do

data/spec/spec_helper.rb

@@ -3,6 +3,9 @@
 ENVIRONMENT = "test"
 APP_ROOT = File.expand_path(File.dirname(__FILE__) + '/../')
 
+require 'coveralls'
+Coveralls.wear!
+
 require "#{APP_ROOT}/lib/cohesion"
 require 'rake'
 

metadata

@@ -1,38 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: cohesion
 version: !ruby/object:Gem::Version
-  version: 0.0.6
-  prerelease: 
+  version: 0.0.7
 platform: ruby
 authors:
 - Stewart McKee
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-14 00:00:00.000000000 Z
+date: 2013-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cobweb
-  requirement: &70099176390280 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099176390280
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ptools
-  requirement: &70099176389260 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099176389260
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Gem to test the cohesion of links within a rails site.  The gem crawls
   the site and checks that external and internal links are valid
 email:
@@ -44,7 +49,8 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .rspec
-- .rvmrc
+- .ruby-gemset
+- .ruby-version
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -61,27 +67,26 @@ files:
 - spec/spec_helper.rb
 homepage: http://github.com/stewartmckee/cohesion
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Gem to test the cohesion of links within a rails site.
 test_files:
 - spec/lib/cohesion_spec.rb

data/.rvmrc

@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-
-# This is an RVM Project .rvmrc file, used to automatically load the ruby
-# development environment upon cd'ing into the directory
-
-# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
-# Only full ruby name is supported here, for short names use:
-#     echo "rvm use 1.9.3" > .rvmrc
-environment_id="ruby-1.9.3@cohesion"
-
-# Uncomment the following lines if you want to verify rvm version per project
-# rvmrc_rvm_version="1.10.3" # 1.10.1 seams as a safe start
-# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
-#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
-#   return 1
-# }
-
-# First we attempt to load the desired environment directly from the environment
-# file. This is very fast and efficient compared to running through the entire
-# CLI and selector. If you want feedback on which environment was used then
-# insert the word 'use' after --create as this triggers verbose mode.
-if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
-  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
-then
-  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
-  [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
-    \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
-  if [[ $- == *i* ]] # check for interactive shells
-  then echo "Using: $(tput setaf 2)$GEM_HOME$(tput sgr0)" # show the user the ruby and gemset they are using in green
-  else echo "Using: $GEM_HOME" # don't use colors in non-interactive shells
-  fi
-else
-  # If the environment file has not yet been created, use the RVM CLI to select.
-  rvm --create use  "$environment_id" || {
-    echo "Failed to create RVM environment '${environment_id}'."
-    return 1
-  }
-fi
-
-# If you use bundler, this might be useful to you:
-# if [[ -s Gemfile ]] && {
-#   ! builtin command -v bundle >/dev/null ||
-#   builtin command -v bundle | grep $rvm_path/bin/bundle >/dev/null
-# }
-# then
-#   printf "%b" "The rubygem 'bundler' is not installed. Installing it now.\n"
-#   gem install bundler
-# fi
-# if [[ -s Gemfile ]] && builtin command -v bundle >/dev/null
-# then
-#   bundle install | grep -vE '^Using|Your bundle is complete'
-# fi