sengi 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (34) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +7 -0
  3. data/Gemfile +3 -0
  4. data/Gemfile.lock +71 -0
  5. data/Makefile +23 -0
  6. data/Makefile.common +58 -0
  7. data/README.md +59 -0
  8. data/Rakefile +25 -0
  9. data/bin/config +148 -0
  10. data/bin/crawler +64 -0
  11. data/bin/list +129 -0
  12. data/bin/redis_start +11 -0
  13. data/bin/redis_stats +13 -0
  14. data/bin/redis_stop +10 -0
  15. data/bin/resque_crawler_restart +14 -0
  16. data/bin/resque_crawler_start +21 -0
  17. data/bin/resque_crawler_stop +20 -0
  18. data/bin/resque_scheduler_start +15 -0
  19. data/bin/resque_scheduler_stop +16 -0
  20. data/bin/resque_server_start +13 -0
  21. data/bin/resque_server_stop +13 -0
  22. data/config/redis.conf +120 -0
  23. data/config/resque_server_config.rb +6 -0
  24. data/lib/sengi.rb +5 -0
  25. data/lib/sengi/crawler.rb +589 -0
  26. data/lib/sengi/crawler_worker.rb +16 -0
  27. data/lib/sengi/uri.rb +288 -0
  28. data/lib/sengi/version.rb +17 -0
  29. data/sengi.gemspec +37 -0
  30. data/sengi.sublime-project +10 -0
  31. data/tests/tc_crawler.rb +14 -0
  32. data/tests/tc_uri.rb +140 -0
  33. data/tests/ts_all.rb +4 -0
  34. metadata +202 -0
data/tests/ts_all.rb ADDED
@@ -0,0 +1,4 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative 'tc_uri'
4
+ require_relative 'tc_crawler'
metadata ADDED
@@ -0,0 +1,202 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sengi
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Christian Mayer
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2016-05-07 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: minitest
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '5.8'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '5.8'
27
+ - !ruby/object:Gem::Dependency
28
+ name: activesupport
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '4.2'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '4.2'
41
+ - !ruby/object:Gem::Dependency
42
+ name: redis
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.2'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.2'
55
+ - !ruby/object:Gem::Dependency
56
+ name: hiredis
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '0.6'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '0.6'
69
+ - !ruby/object:Gem::Dependency
70
+ name: resque
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '1.26'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '1.26'
83
+ - !ruby/object:Gem::Dependency
84
+ name: resque-scheduler
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '4.1'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '4.1'
97
+ - !ruby/object:Gem::Dependency
98
+ name: nokogiri
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '1.6'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '1.6'
111
+ - !ruby/object:Gem::Dependency
112
+ name: cookiejar
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.3'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.3'
125
+ - !ruby/object:Gem::Dependency
126
+ name: thefox-ext
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '1.4'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '1.4'
139
+ description: A web crawler using Ruby and Redis.
140
+ email: christian@fox21.at
141
+ executables: []
142
+ extensions: []
143
+ extra_rdoc_files: []
144
+ files:
145
+ - ".gitignore"
146
+ - Gemfile
147
+ - Gemfile.lock
148
+ - Makefile
149
+ - Makefile.common
150
+ - README.md
151
+ - Rakefile
152
+ - bin/config
153
+ - bin/crawler
154
+ - bin/list
155
+ - bin/redis_start
156
+ - bin/redis_stats
157
+ - bin/redis_stop
158
+ - bin/resque_crawler_restart
159
+ - bin/resque_crawler_start
160
+ - bin/resque_crawler_stop
161
+ - bin/resque_scheduler_start
162
+ - bin/resque_scheduler_stop
163
+ - bin/resque_server_start
164
+ - bin/resque_server_stop
165
+ - config/redis.conf
166
+ - config/resque_server_config.rb
167
+ - lib/sengi.rb
168
+ - lib/sengi/crawler.rb
169
+ - lib/sengi/crawler_worker.rb
170
+ - lib/sengi/uri.rb
171
+ - lib/sengi/version.rb
172
+ - sengi.gemspec
173
+ - sengi.sublime-project
174
+ - tests/tc_crawler.rb
175
+ - tests/tc_uri.rb
176
+ - tests/ts_all.rb
177
+ homepage: https://github.com/TheFox/sengi
178
+ licenses:
179
+ - GPL-3.0
180
+ metadata: {}
181
+ post_install_message:
182
+ rdoc_options: []
183
+ require_paths:
184
+ - lib
185
+ required_ruby_version: !ruby/object:Gem::Requirement
186
+ requirements:
187
+ - - ">="
188
+ - !ruby/object:Gem::Version
189
+ version: 2.1.0
190
+ required_rubygems_version: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - ">="
193
+ - !ruby/object:Gem::Version
194
+ version: '0'
195
+ requirements: []
196
+ rubyforge_project:
197
+ rubygems_version: 2.4.7
198
+ signing_key:
199
+ specification_version: 4
200
+ summary: Sengi Web Crawler
201
+ test_files: []
202
+ has_rdoc: