cohesion 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- data/.rvmrc +1 -1
- data/Gemfile +1 -1
- data/Gemfile.lock +6 -5
- data/README.textile +30 -1
- data/bin/cohesion +21 -8
- data/lib/cohesion.rb +5 -2
- data/lib/cohesion/version.rb +1 -1
- metadata +6 -6
data/.rvmrc
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
|
7
7
|
# Only full ruby name is supported here, for short names use:
|
8
8
|
# echo "rvm use 1.9.3" > .rvmrc
|
9
|
-
environment_id="ruby-1.9.3
|
9
|
+
environment_id="ruby-1.9.3@cohesion"
|
10
10
|
|
11
11
|
# Uncomment the following lines if you want to verify rvm version per project
|
12
12
|
# rvmrc_rvm_version="1.10.3" # 1.10.1 seams as a safe start
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
cohesion (0.0.
|
4
|
+
cohesion (0.0.5)
|
5
5
|
cobweb
|
6
6
|
ptools
|
7
7
|
|
@@ -10,7 +10,7 @@ GEM
|
|
10
10
|
specs:
|
11
11
|
addressable (2.3.2)
|
12
12
|
awesome_print (1.1.0)
|
13
|
-
cobweb (1.0.
|
13
|
+
cobweb (1.0.10)
|
14
14
|
addressable
|
15
15
|
awesome_print
|
16
16
|
haml
|
@@ -25,9 +25,10 @@ GEM
|
|
25
25
|
daemons (1.1.9)
|
26
26
|
diff-lcs (1.1.3)
|
27
27
|
eventmachine (1.0.0)
|
28
|
-
haml (
|
28
|
+
haml (4.0.0)
|
29
|
+
tilt
|
29
30
|
json (1.7.7)
|
30
|
-
multi_json (1.
|
31
|
+
multi_json (1.6.0)
|
31
32
|
namespaced_redis (1.0.4)
|
32
33
|
redis
|
33
34
|
nokogiri (1.5.6)
|
@@ -70,7 +71,7 @@ PLATFORMS
|
|
70
71
|
|
71
72
|
DEPENDENCIES
|
72
73
|
awesome_print
|
73
|
-
cobweb (>= 1.0.
|
74
|
+
cobweb (>= 1.0.10)
|
74
75
|
cohesion!
|
75
76
|
ptools
|
76
77
|
rspec
|
data/README.textile
CHANGED
@@ -21,7 +21,36 @@ h3. Executable
|
|
21
21
|
|
22
22
|
Running from the command line has the following syntax
|
23
23
|
|
24
|
-
bc. cohesion
|
24
|
+
bc. cohesion --url http://your-site-to-check.com/
|
25
|
+
|
26
|
+
|
27
|
+
h4. Options
|
28
|
+
|
29
|
+
Cohesion allows you to control the crawl through various command line options. These are available through help (see below) and are listed here.
|
30
|
+
|
31
|
+
|
32
|
+
* --url <start_url> Url to start crawl from
|
33
|
+
* --internal_urls <url1,url2> List of url patterns to include
|
34
|
+
* --external_urls <exclude_url1, exclude_url2> List of url patterns to exclude
|
35
|
+
* --seed_urls <seed_url1,seed_url2> Seed urls
|
36
|
+
* --crawl_limit <number_of_urls> Limit the crawl to a number of urls
|
37
|
+
* --thread_count <number_of_threads> Set the number of threads used
|
38
|
+
* --timeout <timeout_in_seconds> Sets the timeout for http requests
|
39
|
+
* --output <filename> Path to output data to
|
40
|
+
|
41
|
+
* -v, --verbose Display crawl information
|
42
|
+
* -d, --debug Display debug information
|
43
|
+
* -w, --web_statistics Start web stats server
|
44
|
+
|
45
|
+
|
46
|
+
|
47
|
+
h4. Help
|
48
|
+
|
49
|
+
Help with available options is available with --help
|
50
|
+
|
51
|
+
bc. cohesion --help
|
52
|
+
|
53
|
+
|
25
54
|
|
26
55
|
|
27
56
|
h3. Checking an external site
|
data/bin/cohesion
CHANGED
@@ -7,24 +7,37 @@ require 'cohesion'
|
|
7
7
|
require 'slop'
|
8
8
|
|
9
9
|
opts = Slop.parse(:help => true) do
|
10
|
-
banner 'Usage: cohesion
|
10
|
+
banner 'Usage: cohesion [options]'
|
11
|
+
|
12
|
+
on 'url=', 'URL to start crawl from'
|
13
|
+
on 'internal_urls=', 'Url patterns to include', :as => Array
|
14
|
+
on 'external_urls=', 'Url patterns to exclude', :as => Array
|
15
|
+
on 'seed_urls=', "Seed urls", :as => Array
|
16
|
+
on 'crawl_limit=', 'Limit the crawl to a number of urls', :as => Integer
|
17
|
+
on 'thread_count=', "Set the number of threads used", :as => Integer
|
18
|
+
on 'timeout=', "Sets the timeout for http requests", :as => Integer
|
19
|
+
|
20
|
+
on 'output=', 'Path to output data to'
|
11
21
|
|
12
|
-
on 'u', 'url', 'URL to start crawl from'
|
13
|
-
on 'i', 'internal_urls', 'Url patterns to include', :as => Array
|
14
|
-
on 'e', 'external_urls', 'Url patterns to exclude', :as => Array
|
15
|
-
on 's', 'seed_urls', "Seed urls", :as => Array
|
16
22
|
on 'v', 'verbose', 'Display crawl information'
|
17
23
|
on 'd', 'debug', 'Display debug information'
|
18
|
-
on '
|
19
|
-
on 't', 'thread_count', "Set the number of threads used"
|
24
|
+
on 'w', 'web_statistics', 'Start web stats server'
|
20
25
|
end
|
21
26
|
|
22
27
|
if opts[:url]
|
23
|
-
failures = Cohesion::Check.site(opts[:url], opts)
|
24
28
|
|
29
|
+
options = opts.to_hash.delete_if { |k, v| v.nil? || k == :url}
|
30
|
+
ap options
|
31
|
+
|
32
|
+
failures = Cohesion::Check.site(opts[:url], options)
|
25
33
|
if failures.count == 0
|
26
34
|
exit(true)
|
27
35
|
else
|
36
|
+
if opts[:output]
|
37
|
+
File.open(opts[:output], 'w') do |f|
|
38
|
+
f.write failures.to_json
|
39
|
+
end
|
40
|
+
end
|
28
41
|
exit(false)
|
29
42
|
end
|
30
43
|
else
|
data/lib/cohesion.rb
CHANGED
@@ -78,14 +78,17 @@ module Cohesion
|
|
78
78
|
end
|
79
79
|
end
|
80
80
|
|
81
|
+
issues = []
|
81
82
|
if failures.count == 0
|
82
83
|
puts "All links working!"
|
83
84
|
else
|
84
85
|
puts "Failed urls:"
|
85
86
|
failures.each do |f|
|
87
|
+
inbound_links = statistics.inbound_links_for(f[:url])
|
88
|
+
issues << {:issue => f, :inbound => inbound_links}
|
86
89
|
puts ""
|
87
90
|
puts "#{f[:url]} [ #{f[:status_code]} ]"
|
88
|
-
|
91
|
+
inbound_links.each do |inbound_link|
|
89
92
|
puts " - #{inbound_link}"
|
90
93
|
end
|
91
94
|
end
|
@@ -93,7 +96,7 @@ module Cohesion
|
|
93
96
|
end
|
94
97
|
puts
|
95
98
|
|
96
|
-
|
99
|
+
return issues
|
97
100
|
end
|
98
101
|
end
|
99
102
|
end
|
data/lib/cohesion/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cohesion
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-02-
|
12
|
+
date: 2013-02-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: cobweb
|
16
|
-
requirement: &
|
16
|
+
requirement: &70099176390280 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70099176390280
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: ptools
|
27
|
-
requirement: &
|
27
|
+
requirement: &70099176389260 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,7 +32,7 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70099176389260
|
36
36
|
description: Gem to test the cohesion of links within a rails site. The gem crawls
|
37
37
|
the site and checks that external and internal links are valid
|
38
38
|
email:
|