cobweb 0.0.26 → 0.0.27

Sign up to get free protection for your applications and to get access to all the features.
Files changed (4) hide show
  1. data/README.textile +1 -1
  2. data/lib/cobweb.rb +1 -1
  3. data/lib/crawl_job.rb +1 -1
  4. metadata +20 -20
data/README.textile CHANGED
@@ -1,5 +1,5 @@
1
1
 
2
- h1. Cobweb v0.0.26
2
+ h1. Cobweb v0.0.27
3
3
 
4
4
  h2. Intro
5
5
 
data/lib/cobweb.rb CHANGED
@@ -19,7 +19,7 @@ class Cobweb
19
19
  # investigate using event machine for single threaded crawling
20
20
 
21
21
  def self.version
22
- "0.0.26"
22
+ "0.0.27"
23
23
  end
24
24
 
25
25
  def method_missing(method_sym, *arguments, &block)
data/lib/crawl_job.rb CHANGED
@@ -85,7 +85,7 @@ class CrawlJob
85
85
  private
86
86
  def self.set_base_url(redis, content, content_request)
87
87
  if redis.get("base_url").nil?
88
- unless content[:redirect_through].empty? || !content_request[:first_page_redirect_internal]
88
+ unless content[:redirect_through].nil? || content[:redirect_through].empty? || !content_request[:first_page_redirect_internal]
89
89
  uri = Addressable::URI.parse(content[:redirect_through].last)
90
90
  redis.sadd("internal_urls", [uri.scheme, "://", uri.host, "/*"].join)
91
91
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cobweb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.26
4
+ version: 0.0.27
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-03-13 00:00:00.000000000 Z
12
+ date: 2012-03-14 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: resque
16
- requirement: &70294818287920 !ruby/object:Gem::Requirement
16
+ requirement: &70097319100920 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: '0'
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *70294818287920
24
+ version_requirements: *70097319100920
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: redis
27
- requirement: &70294818287500 !ruby/object:Gem::Requirement
27
+ requirement: &70097319100360 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
32
32
  version: '0'
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *70294818287500
35
+ version_requirements: *70097319100360
36
36
  - !ruby/object:Gem::Dependency
37
37
  name: nokogiri
38
- requirement: &70294818287080 !ruby/object:Gem::Requirement
38
+ requirement: &70097319099940 !ruby/object:Gem::Requirement
39
39
  none: false
40
40
  requirements:
41
41
  - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
43
43
  version: '0'
44
44
  type: :runtime
45
45
  prerelease: false
46
- version_requirements: *70294818287080
46
+ version_requirements: *70097319099940
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: addressable
49
- requirement: &70294818286640 !ruby/object:Gem::Requirement
49
+ requirement: &70097319099500 !ruby/object:Gem::Requirement
50
50
  none: false
51
51
  requirements:
52
52
  - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
54
54
  version: '0'
55
55
  type: :runtime
56
56
  prerelease: false
57
- version_requirements: *70294818286640
57
+ version_requirements: *70097319099500
58
58
  - !ruby/object:Gem::Dependency
59
59
  name: rspec
60
- requirement: &70294818286220 !ruby/object:Gem::Requirement
60
+ requirement: &70097319099080 !ruby/object:Gem::Requirement
61
61
  none: false
62
62
  requirements:
63
63
  - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
65
65
  version: '0'
66
66
  type: :runtime
67
67
  prerelease: false
68
- version_requirements: *70294818286220
68
+ version_requirements: *70097319099080
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: awesome_print
71
- requirement: &70294818285800 !ruby/object:Gem::Requirement
71
+ requirement: &70097319098660 !ruby/object:Gem::Requirement
72
72
  none: false
73
73
  requirements:
74
74
  - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
76
76
  version: '0'
77
77
  type: :runtime
78
78
  prerelease: false
79
- version_requirements: *70294818285800
79
+ version_requirements: *70097319098660
80
80
  - !ruby/object:Gem::Dependency
81
81
  name: sinatra
82
- requirement: &70294818285380 !ruby/object:Gem::Requirement
82
+ requirement: &70097319098240 !ruby/object:Gem::Requirement
83
83
  none: false
84
84
  requirements:
85
85
  - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
87
87
  version: '0'
88
88
  type: :runtime
89
89
  prerelease: false
90
- version_requirements: *70294818285380
90
+ version_requirements: *70097319098240
91
91
  - !ruby/object:Gem::Dependency
92
92
  name: thin
93
- requirement: &70294818284960 !ruby/object:Gem::Requirement
93
+ requirement: &70097319097820 !ruby/object:Gem::Requirement
94
94
  none: false
95
95
  requirements:
96
96
  - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
98
98
  version: '0'
99
99
  type: :runtime
100
100
  prerelease: false
101
- version_requirements: *70294818284960
101
+ version_requirements: *70097319097820
102
102
  - !ruby/object:Gem::Dependency
103
103
  name: haml
104
- requirement: &70294818284540 !ruby/object:Gem::Requirement
104
+ requirement: &70097319097400 !ruby/object:Gem::Requirement
105
105
  none: false
106
106
  requirements:
107
107
  - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
109
109
  version: '0'
110
110
  type: :runtime
111
111
  prerelease: false
112
- version_requirements: *70294818284540
112
+ version_requirements: *70097319097400
113
113
  description: Web Crawler that uses resque background job engine to allow you to cluster
114
114
  your crawl.
115
115
  email: stewart@rockwellcottage.com