cohesion 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.rvmrc +1 -1
- data/Gemfile +1 -1
- data/Gemfile.lock +6 -5
- data/README.textile +30 -1
- data/bin/cohesion +21 -8
- data/lib/cohesion.rb +5 -2
- data/lib/cohesion/version.rb +1 -1
- metadata +6 -6
data/.rvmrc
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
|
7
7
|
# Only full ruby name is supported here, for short names use:
|
8
8
|
# echo "rvm use 1.9.3" > .rvmrc
|
9
|
-
environment_id="ruby-1.9.3
|
9
|
+
environment_id="ruby-1.9.3@cohesion"
|
10
10
|
|
11
11
|
# Uncomment the following lines if you want to verify rvm version per project
|
12
12
|
# rvmrc_rvm_version="1.10.3" # 1.10.1 seams as a safe start
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
cohesion (0.0.
|
4
|
+
cohesion (0.0.5)
|
5
5
|
cobweb
|
6
6
|
ptools
|
7
7
|
|
@@ -10,7 +10,7 @@ GEM
|
|
10
10
|
specs:
|
11
11
|
addressable (2.3.2)
|
12
12
|
awesome_print (1.1.0)
|
13
|
-
cobweb (1.0.
|
13
|
+
cobweb (1.0.10)
|
14
14
|
addressable
|
15
15
|
awesome_print
|
16
16
|
haml
|
@@ -25,9 +25,10 @@ GEM
|
|
25
25
|
daemons (1.1.9)
|
26
26
|
diff-lcs (1.1.3)
|
27
27
|
eventmachine (1.0.0)
|
28
|
-
haml (
|
28
|
+
haml (4.0.0)
|
29
|
+
tilt
|
29
30
|
json (1.7.7)
|
30
|
-
multi_json (1.
|
31
|
+
multi_json (1.6.0)
|
31
32
|
namespaced_redis (1.0.4)
|
32
33
|
redis
|
33
34
|
nokogiri (1.5.6)
|
@@ -70,7 +71,7 @@ PLATFORMS
|
|
70
71
|
|
71
72
|
DEPENDENCIES
|
72
73
|
awesome_print
|
73
|
-
cobweb (>= 1.0.
|
74
|
+
cobweb (>= 1.0.10)
|
74
75
|
cohesion!
|
75
76
|
ptools
|
76
77
|
rspec
|
data/README.textile
CHANGED
@@ -21,7 +21,36 @@ h3. Executable
|
|
21
21
|
|
22
22
|
Running from the command line has the following syntax
|
23
23
|
|
24
|
-
bc. cohesion
|
24
|
+
bc. cohesion --url http://your-site-to-check.com/
|
25
|
+
|
26
|
+
|
27
|
+
h4. Options
|
28
|
+
|
29
|
+
Cohesion allows you to control the crawl through various command line options. These are available through help (see below) and are listed here.
|
30
|
+
|
31
|
+
|
32
|
+
* --url <start_url> Url to start crawl from
|
33
|
+
* --internal_urls <url1,url2> List of url patterns to include
|
34
|
+
* --external_urls <exclude_url1, exclude_url2> List of url patterns to exclude
|
35
|
+
* --seed_urls <seed_url1,seed_url2> Seed urls
|
36
|
+
* --crawl_limit <number_of_urls> Limit the crawl to a number of urls
|
37
|
+
* --thread_count <number_of_threads> Set the number of threads used
|
38
|
+
* --timeout <timeout_in_seconds> Sets the timeout for http requests
|
39
|
+
* --output <filename> Path to output data to
|
40
|
+
|
41
|
+
* -v, --verbose Display crawl information
|
42
|
+
* -d, --debug Display debug information
|
43
|
+
* -w, --web_statistics Start web stats server
|
44
|
+
|
45
|
+
|
46
|
+
|
47
|
+
h4. Help
|
48
|
+
|
49
|
+
Help with available options is available with --help
|
50
|
+
|
51
|
+
bc. cohesion --help
|
52
|
+
|
53
|
+
|
25
54
|
|
26
55
|
|
27
56
|
h3. Checking an external site
|
data/bin/cohesion
CHANGED
@@ -7,24 +7,37 @@ require 'cohesion'
|
|
7
7
|
require 'slop'
|
8
8
|
|
9
9
|
opts = Slop.parse(:help => true) do
|
10
|
-
banner 'Usage: cohesion
|
10
|
+
banner 'Usage: cohesion [options]'
|
11
|
+
|
12
|
+
on 'url=', 'URL to start crawl from'
|
13
|
+
on 'internal_urls=', 'Url patterns to include', :as => Array
|
14
|
+
on 'external_urls=', 'Url patterns to exclude', :as => Array
|
15
|
+
on 'seed_urls=', "Seed urls", :as => Array
|
16
|
+
on 'crawl_limit=', 'Limit the crawl to a number of urls', :as => Integer
|
17
|
+
on 'thread_count=', "Set the number of threads used", :as => Integer
|
18
|
+
on 'timeout=', "Sets the timeout for http requests", :as => Integer
|
19
|
+
|
20
|
+
on 'output=', 'Path to output data to'
|
11
21
|
|
12
|
-
on 'u', 'url', 'URL to start crawl from'
|
13
|
-
on 'i', 'internal_urls', 'Url patterns to include', :as => Array
|
14
|
-
on 'e', 'external_urls', 'Url patterns to exclude', :as => Array
|
15
|
-
on 's', 'seed_urls', "Seed urls", :as => Array
|
16
22
|
on 'v', 'verbose', 'Display crawl information'
|
17
23
|
on 'd', 'debug', 'Display debug information'
|
18
|
-
on '
|
19
|
-
on 't', 'thread_count', "Set the number of threads used"
|
24
|
+
on 'w', 'web_statistics', 'Start web stats server'
|
20
25
|
end
|
21
26
|
|
22
27
|
if opts[:url]
|
23
|
-
failures = Cohesion::Check.site(opts[:url], opts)
|
24
28
|
|
29
|
+
options = opts.to_hash.delete_if { |k, v| v.nil? || k == :url}
|
30
|
+
ap options
|
31
|
+
|
32
|
+
failures = Cohesion::Check.site(opts[:url], options)
|
25
33
|
if failures.count == 0
|
26
34
|
exit(true)
|
27
35
|
else
|
36
|
+
if opts[:output]
|
37
|
+
File.open(opts[:output], 'w') do |f|
|
38
|
+
f.write failures.to_json
|
39
|
+
end
|
40
|
+
end
|
28
41
|
exit(false)
|
29
42
|
end
|
30
43
|
else
|
data/lib/cohesion.rb
CHANGED
@@ -78,14 +78,17 @@ module Cohesion
|
|
78
78
|
end
|
79
79
|
end
|
80
80
|
|
81
|
+
issues = []
|
81
82
|
if failures.count == 0
|
82
83
|
puts "All links working!"
|
83
84
|
else
|
84
85
|
puts "Failed urls:"
|
85
86
|
failures.each do |f|
|
87
|
+
inbound_links = statistics.inbound_links_for(f[:url])
|
88
|
+
issues << {:issue => f, :inbound => inbound_links}
|
86
89
|
puts ""
|
87
90
|
puts "#{f[:url]} [ #{f[:status_code]} ]"
|
88
|
-
|
91
|
+
inbound_links.each do |inbound_link|
|
89
92
|
puts " - #{inbound_link}"
|
90
93
|
end
|
91
94
|
end
|
@@ -93,7 +96,7 @@ module Cohesion
|
|
93
96
|
end
|
94
97
|
puts
|
95
98
|
|
96
|
-
|
99
|
+
return issues
|
97
100
|
end
|
98
101
|
end
|
99
102
|
end
|
data/lib/cohesion/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cohesion
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-02-
|
12
|
+
date: 2013-02-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: cobweb
|
16
|
-
requirement: &
|
16
|
+
requirement: &70099176390280 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70099176390280
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: ptools
|
27
|
-
requirement: &
|
27
|
+
requirement: &70099176389260 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,7 +32,7 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70099176389260
|
36
36
|
description: Gem to test the cohesion of links within a rails site. The gem crawls
|
37
37
|
the site and checks that external and internal links are valid
|
38
38
|
email:
|