canonicurl 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/canonicurl.gemspec +0 -1
- data/lib/canonicurl/cache.rb +18 -40
- data/lib/canonicurl/version.rb +1 -1
- metadata +4 -22
data/canonicurl.gemspec
CHANGED
data/lib/canonicurl/cache.rb
CHANGED
@@ -36,7 +36,7 @@ module Canonicurl
|
|
36
36
|
end
|
37
37
|
|
38
38
|
|
39
|
-
def fetch(url,
|
39
|
+
def fetch(url, resolver)
|
40
40
|
k = key(url)
|
41
41
|
@db.setnx(k, LOCKED) # lock it if key doesn't exist
|
42
42
|
|
@@ -49,8 +49,7 @@ module Canonicurl
|
|
49
49
|
when CANONICAL
|
50
50
|
url
|
51
51
|
when LOCKED
|
52
|
-
resolve(url, k,
|
53
|
-
RESOLVING
|
52
|
+
resolve(url, k, resolver)
|
54
53
|
else
|
55
54
|
result
|
56
55
|
end
|
@@ -75,49 +74,28 @@ module Canonicurl
|
|
75
74
|
|
76
75
|
private
|
77
76
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
em_already_running = running
|
83
|
-
http = EM::HttpRequest.new(url,
|
84
|
-
:connection_timeout => @timeout,
|
85
|
-
:inactivity_timeout => @timeout * 2).get(:redirects => @redirects)
|
86
|
-
http.callback {
|
87
|
-
status = http.response_header.status.to_i
|
88
|
-
case status
|
89
|
-
when 200...300
|
90
|
-
http.last_effective_url.port = nil if http.last_effective_url.port == 80 # don't show default port 80
|
91
|
-
canonical_url = http.last_effective_url.to_s
|
92
|
-
set url, canonical_url, url_key
|
93
|
-
callbacks[:resolved].call(canonical_url, http) if callbacks[:resolved]
|
94
|
-
else
|
95
|
-
@db.set url_key, (status / 100).to_s
|
96
|
-
callbacks[:failed].call(http) if callbacks[:failed]
|
97
|
-
end
|
98
|
-
EM.stop unless em_already_running
|
99
|
-
}
|
100
|
-
http.errback {
|
101
|
-
@db.set(url_key, ERROR)
|
102
|
-
callbacks[:error].call(http) if callbacks[:error]
|
103
|
-
EM.stop unless em_already_running
|
104
|
-
}
|
105
|
-
end
|
77
|
+
def resolve(url, url_key, resolver)
|
78
|
+
@db.set(url_key, RESOLVING)
|
79
|
+
begin
|
80
|
+
canonical_url = resolver.call(url).to_s
|
106
81
|
rescue Exception => e
|
107
82
|
@db.set(url_key, ERROR)
|
108
|
-
|
109
|
-
EM.stop unless em_already_running
|
83
|
+
raise e
|
110
84
|
end
|
111
85
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
86
|
+
if canonical_url.size > 1
|
87
|
+
set url, canonical_url, url_key
|
88
|
+
canonical_url
|
89
|
+
else
|
90
|
+
if canonical_url.size == 1 && ![CANONICAL, ERROR, LOCKED, RESOLVING].include?(canonical_url)
|
91
|
+
@db.set(url_key, canonical_url) # save status
|
92
|
+
canonical_url
|
116
93
|
else
|
117
|
-
|
118
|
-
|
119
|
-
end
|
94
|
+
@db.set(url_key, ERROR) # error
|
95
|
+
ERROR
|
120
96
|
end
|
121
97
|
end
|
98
|
+
end
|
99
|
+
|
122
100
|
end
|
123
101
|
end
|
data/lib/canonicurl/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: canonicurl
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 19
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 0
|
9
|
-
-
|
10
|
-
version: 0.0.
|
9
|
+
- 6
|
10
|
+
version: 0.0.6
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- David Dai
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-04-
|
18
|
+
date: 2011-04-07 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
name: redis
|
@@ -33,24 +33,6 @@ dependencies:
|
|
33
33
|
version: 2.2.0
|
34
34
|
type: :runtime
|
35
35
|
version_requirements: *id001
|
36
|
-
- !ruby/object:Gem::Dependency
|
37
|
-
name: em-http-request
|
38
|
-
prerelease: false
|
39
|
-
requirement: &id002 !ruby/object:Gem::Requirement
|
40
|
-
none: false
|
41
|
-
requirements:
|
42
|
-
- - ">="
|
43
|
-
- !ruby/object:Gem::Version
|
44
|
-
hash: 62196357
|
45
|
-
segments:
|
46
|
-
- 1
|
47
|
-
- 0
|
48
|
-
- 0
|
49
|
-
- beta
|
50
|
-
- 3
|
51
|
-
version: 1.0.0.beta.3
|
52
|
-
type: :runtime
|
53
|
-
version_requirements: *id002
|
54
36
|
description: ""
|
55
37
|
email:
|
56
38
|
- david.github@gmail.com
|