cobweb 0.0.11 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.textile +8 -0
- metadata +13 -12
data/README.textile
CHANGED
|
@@ -78,6 +78,14 @@ Simple get that obey's the options supplied in new.
|
|
|
78
78
|
bq. crawler.head("http://www.google.com/")
|
|
79
79
|
|
|
80
80
|
|
|
81
|
+
h2. Planned Work
|
|
82
|
+
|
|
83
|
+
These features are what I'd like to see added to cobweb. If your really interested in any of these features give me a ping, it will give me a good excuse to get them done!
|
|
84
|
+
|
|
85
|
+
* remove resque requirement - I'd like the crawler to work independant of resque as a normal standalone crawler
|
|
86
|
+
* fix crawl finished notification
|
|
87
|
+
* improve internal link control
|
|
88
|
+
|
|
81
89
|
h2. License
|
|
82
90
|
|
|
83
91
|
h3. The MIT License
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: cobweb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.12
|
|
5
5
|
prerelease:
|
|
6
6
|
platform: ruby
|
|
7
7
|
authors:
|
|
@@ -13,7 +13,7 @@ date: 2012-01-15 00:00:00.000000000 Z
|
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: resque
|
|
16
|
-
requirement: &
|
|
16
|
+
requirement: &70227287202120 !ruby/object:Gem::Requirement
|
|
17
17
|
none: false
|
|
18
18
|
requirements:
|
|
19
19
|
- - ! '>='
|
|
@@ -21,10 +21,10 @@ dependencies:
|
|
|
21
21
|
version: '0'
|
|
22
22
|
type: :runtime
|
|
23
23
|
prerelease: false
|
|
24
|
-
version_requirements: *
|
|
24
|
+
version_requirements: *70227287202120
|
|
25
25
|
- !ruby/object:Gem::Dependency
|
|
26
26
|
name: redis
|
|
27
|
-
requirement: &
|
|
27
|
+
requirement: &70227287201020 !ruby/object:Gem::Requirement
|
|
28
28
|
none: false
|
|
29
29
|
requirements:
|
|
30
30
|
- - ! '>='
|
|
@@ -32,10 +32,10 @@ dependencies:
|
|
|
32
32
|
version: '0'
|
|
33
33
|
type: :runtime
|
|
34
34
|
prerelease: false
|
|
35
|
-
version_requirements: *
|
|
35
|
+
version_requirements: *70227287201020
|
|
36
36
|
- !ruby/object:Gem::Dependency
|
|
37
37
|
name: absolutize
|
|
38
|
-
requirement: &
|
|
38
|
+
requirement: &70227287200100 !ruby/object:Gem::Requirement
|
|
39
39
|
none: false
|
|
40
40
|
requirements:
|
|
41
41
|
- - ! '>='
|
|
@@ -43,10 +43,10 @@ dependencies:
|
|
|
43
43
|
version: '0'
|
|
44
44
|
type: :runtime
|
|
45
45
|
prerelease: false
|
|
46
|
-
version_requirements: *
|
|
46
|
+
version_requirements: *70227287200100
|
|
47
47
|
- !ruby/object:Gem::Dependency
|
|
48
48
|
name: nokogiri
|
|
49
|
-
requirement: &
|
|
49
|
+
requirement: &70227287199600 !ruby/object:Gem::Requirement
|
|
50
50
|
none: false
|
|
51
51
|
requirements:
|
|
52
52
|
- - ! '>='
|
|
@@ -54,10 +54,10 @@ dependencies:
|
|
|
54
54
|
version: '0'
|
|
55
55
|
type: :runtime
|
|
56
56
|
prerelease: false
|
|
57
|
-
version_requirements: *
|
|
57
|
+
version_requirements: *70227287199600
|
|
58
58
|
- !ruby/object:Gem::Dependency
|
|
59
59
|
name: addressable
|
|
60
|
-
requirement: &
|
|
60
|
+
requirement: &70227287198900 !ruby/object:Gem::Requirement
|
|
61
61
|
none: false
|
|
62
62
|
requirements:
|
|
63
63
|
- - ! '>='
|
|
@@ -65,7 +65,7 @@ dependencies:
|
|
|
65
65
|
version: '0'
|
|
66
66
|
type: :runtime
|
|
67
67
|
prerelease: false
|
|
68
|
-
version_requirements: *
|
|
68
|
+
version_requirements: *70227287198900
|
|
69
69
|
description:
|
|
70
70
|
email: stewart@rockwellcottage.com
|
|
71
71
|
executables: []
|
|
@@ -108,5 +108,6 @@ rubyforge_project:
|
|
|
108
108
|
rubygems_version: 1.8.11
|
|
109
109
|
signing_key:
|
|
110
110
|
specification_version: 3
|
|
111
|
-
summary: Crawler
|
|
111
|
+
summary: Web Crawler that uses resque background job engine to allow you to cluster
|
|
112
|
+
your crawl.
|
|
112
113
|
test_files: []
|