web_crawler 0.3.1 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (33) hide show
  1. data/Gemfile +2 -0
  2. data/README +22 -1
  3. data/lib/web_crawler.rb +2 -0
  4. data/lib/web_crawler/application.rb +33 -2
  5. data/lib/web_crawler/base.rb +113 -0
  6. data/lib/web_crawler/batch_request.rb +10 -4
  7. data/lib/web_crawler/cached_request.rb +16 -7
  8. data/lib/web_crawler/configuration.rb +5 -5
  9. data/lib/web_crawler/factory_url.rb +27 -7
  10. data/lib/web_crawler/follower.rb +11 -9
  11. data/lib/web_crawler/parsers.rb +1 -0
  12. data/lib/web_crawler/parsers/mapper.rb +114 -0
  13. data/lib/web_crawler/parsers/url.rb +3 -5
  14. data/lib/web_crawler/request.rb +14 -2
  15. data/lib/web_crawler/response.rb +2 -2
  16. data/lib/web_crawler/version.rb +2 -2
  17. data/lib/web_crawler/view.rb +1 -1
  18. data/lib/web_crawler/view/csv.rb +1 -1
  19. data/lib/web_crawler/view/json.rb +1 -1
  20. data/lib/web_crawler/view/yaml.rb +1 -1
  21. data/spec/fixtures/example.xml +171 -0
  22. data/spec/fixtures/my_crawler.rb +82 -0
  23. data/spec/fixtures/test_crawler.rb +108 -0
  24. data/spec/fixtures/test_crawler2.rb +77 -0
  25. data/spec/spec_helper.rb +8 -3
  26. data/spec/web_crawler/batch_request_spec.rb +0 -11
  27. data/spec/web_crawler/cached_request_spec.rb +17 -11
  28. data/spec/web_crawler/factory_url_spec.rb +19 -6
  29. data/spec/web_crawler/follow_spec.rb +11 -4
  30. data/spec/web_crawler/view_spec.rb +10 -10
  31. data/spec/web_crawler/web_crawler_api_base_class_spec.rb +143 -0
  32. data/web_crawler.gemspec +2 -0
  33. metadata +43 -8
data/web_crawler.gemspec CHANGED
@@ -25,6 +25,8 @@ Gem::Specification.new do |s|
25
25
 
26
26
  s.add_dependency 'thor', '>=0.14.6'
27
27
  s.add_dependency 'mime-types', '>=1.16'
28
+ s.add_dependency 'parallel', '>=0.5.5'
29
+ s.add_dependency 'activesupport'
28
30
 
29
31
  s.add_development_dependency(%q<rspec>, [">=2.6"])
30
32
  s.add_development_dependency(%q<fakeweb>)
metadata CHANGED
@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
4
4
  prerelease: false
5
5
  segments:
6
6
  - 0
7
- - 3
8
- - 1
9
- version: 0.3.1
7
+ - 5
8
+ - 0
9
+ version: 0.5.0
10
10
  platform: ruby
11
11
  authors:
12
12
  - Anton Sozontov
@@ -14,7 +14,7 @@ autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
16
 
17
- date: 2011-06-03 00:00:00 +04:00
17
+ date: 2011-06-14 00:00:00 +04:00
18
18
  default_executable:
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency
@@ -47,9 +47,37 @@ dependencies:
47
47
  type: :runtime
48
48
  version_requirements: *id002
49
49
  - !ruby/object:Gem::Dependency
50
- name: rspec
50
+ name: parallel
51
51
  prerelease: false
52
52
  requirement: &id003 !ruby/object:Gem::Requirement
53
+ none: false
54
+ requirements:
55
+ - - ">="
56
+ - !ruby/object:Gem::Version
57
+ segments:
58
+ - 0
59
+ - 5
60
+ - 5
61
+ version: 0.5.5
62
+ type: :runtime
63
+ version_requirements: *id003
64
+ - !ruby/object:Gem::Dependency
65
+ name: activesupport
66
+ prerelease: false
67
+ requirement: &id004 !ruby/object:Gem::Requirement
68
+ none: false
69
+ requirements:
70
+ - - ">="
71
+ - !ruby/object:Gem::Version
72
+ segments:
73
+ - 0
74
+ version: "0"
75
+ type: :runtime
76
+ version_requirements: *id004
77
+ - !ruby/object:Gem::Dependency
78
+ name: rspec
79
+ prerelease: false
80
+ requirement: &id005 !ruby/object:Gem::Requirement
53
81
  none: false
54
82
  requirements:
55
83
  - - ">="
@@ -59,11 +87,11 @@ dependencies:
59
87
  - 6
60
88
  version: "2.6"
61
89
  type: :development
62
- version_requirements: *id003
90
+ version_requirements: *id005
63
91
  - !ruby/object:Gem::Dependency
64
92
  name: fakeweb
65
93
  prerelease: false
66
- requirement: &id004 !ruby/object:Gem::Requirement
94
+ requirement: &id006 !ruby/object:Gem::Requirement
67
95
  none: false
68
96
  requirements:
69
97
  - - ">="
@@ -72,7 +100,7 @@ dependencies:
72
100
  - 0
73
101
  version: "0"
74
102
  type: :development
75
- version_requirements: *id004
103
+ version_requirements: *id006
76
104
  description: Web crawler help you with parse and collect data from the web
77
105
  email:
78
106
  - a.sozontov@gmail.com
@@ -94,6 +122,7 @@ files:
94
122
  - lib/ext/http_response.rb
95
123
  - lib/web_crawler.rb
96
124
  - lib/web_crawler/application.rb
125
+ - lib/web_crawler/base.rb
97
126
  - lib/web_crawler/batch_request.rb
98
127
  - lib/web_crawler/cache_adapter.rb
99
128
  - lib/web_crawler/cache_adapter/base.rb
@@ -108,6 +137,7 @@ files:
108
137
  - lib/web_crawler/follower.rb
109
138
  - lib/web_crawler/handler.rb
110
139
  - lib/web_crawler/parsers.rb
140
+ - lib/web_crawler/parsers/mapper.rb
111
141
  - lib/web_crawler/parsers/url.rb
112
142
  - lib/web_crawler/request.rb
113
143
  - lib/web_crawler/response.rb
@@ -122,6 +152,10 @@ files:
122
152
  - lib/web_crawler/view/xml.rb
123
153
  - lib/web_crawler/view/yaml.rb
124
154
  - spec/fake_web_generator.rb
155
+ - spec/fixtures/example.xml
156
+ - spec/fixtures/my_crawler.rb
157
+ - spec/fixtures/test_crawler.rb
158
+ - spec/fixtures/test_crawler2.rb
125
159
  - spec/spec_helper.rb
126
160
  - spec/web_crawler/batch_request_spec.rb
127
161
  - spec/web_crawler/cached_request_spec.rb
@@ -131,6 +165,7 @@ files:
131
165
  - spec/web_crawler/response_spec.rb
132
166
  - spec/web_crawler/url_parser_spec.rb
133
167
  - spec/web_crawler/view_spec.rb
168
+ - spec/web_crawler/web_crawler_api_base_class_spec.rb
134
169
  - web_crawler.gemspec
135
170
  has_rdoc: false
136
171
  homepage: ""