wayfarer 0.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (235) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +8 -0
  3. data/.rbenv-gemsets +1 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +21 -0
  6. data/.ruby-version +1 -0
  7. data/.travis.yml +5 -0
  8. data/.yardopts +3 -0
  9. data/Changelog.md +10 -0
  10. data/Gemfile +11 -0
  11. data/LICENSE +19 -0
  12. data/README.md +21 -0
  13. data/Rakefile +114 -0
  14. data/benchmark/frontiers.rb +143 -0
  15. data/bin/wayfarer +116 -0
  16. data/docs/.gitignore +2 -0
  17. data/docs/_config.yml +15 -0
  18. data/docs/_includes/base.html +7 -0
  19. data/docs/_includes/head.html +10 -0
  20. data/docs/_includes/navigation.html +187 -0
  21. data/docs/_layouts/default.html +42 -0
  22. data/docs/_sass/base.scss +439 -0
  23. data/docs/_sass/variables.scss +24 -0
  24. data/docs/_sass/vendor/bourbon/_bourbon-deprecate.scss +19 -0
  25. data/docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss +425 -0
  26. data/docs/_sass/vendor/bourbon/_bourbon.scss +90 -0
  27. data/docs/_sass/vendor/bourbon/addons/_border-color.scss +29 -0
  28. data/docs/_sass/vendor/bourbon/addons/_border-radius.scss +48 -0
  29. data/docs/_sass/vendor/bourbon/addons/_border-style.scss +28 -0
  30. data/docs/_sass/vendor/bourbon/addons/_border-width.scss +28 -0
  31. data/docs/_sass/vendor/bourbon/addons/_buttons.scss +69 -0
  32. data/docs/_sass/vendor/bourbon/addons/_clearfix.scss +25 -0
  33. data/docs/_sass/vendor/bourbon/addons/_ellipsis.scss +30 -0
  34. data/docs/_sass/vendor/bourbon/addons/_font-stacks.scss +31 -0
  35. data/docs/_sass/vendor/bourbon/addons/_hide-text.scss +27 -0
  36. data/docs/_sass/vendor/bourbon/addons/_margin.scss +29 -0
  37. data/docs/_sass/vendor/bourbon/addons/_padding.scss +29 -0
  38. data/docs/_sass/vendor/bourbon/addons/_position.scss +51 -0
  39. data/docs/_sass/vendor/bourbon/addons/_prefixer.scss +66 -0
  40. data/docs/_sass/vendor/bourbon/addons/_retina-image.scss +27 -0
  41. data/docs/_sass/vendor/bourbon/addons/_size.scss +56 -0
  42. data/docs/_sass/vendor/bourbon/addons/_text-inputs.scss +118 -0
  43. data/docs/_sass/vendor/bourbon/addons/_timing-functions.scss +34 -0
  44. data/docs/_sass/vendor/bourbon/addons/_triangle.scss +63 -0
  45. data/docs/_sass/vendor/bourbon/addons/_word-wrap.scss +29 -0
  46. data/docs/_sass/vendor/bourbon/css3/_animation.scss +61 -0
  47. data/docs/_sass/vendor/bourbon/css3/_appearance.scss +5 -0
  48. data/docs/_sass/vendor/bourbon/css3/_backface-visibility.scss +5 -0
  49. data/docs/_sass/vendor/bourbon/css3/_background-image.scss +44 -0
  50. data/docs/_sass/vendor/bourbon/css3/_background.scss +57 -0
  51. data/docs/_sass/vendor/bourbon/css3/_border-image.scss +61 -0
  52. data/docs/_sass/vendor/bourbon/css3/_calc.scss +6 -0
  53. data/docs/_sass/vendor/bourbon/css3/_columns.scss +67 -0
  54. data/docs/_sass/vendor/bourbon/css3/_filter.scss +6 -0
  55. data/docs/_sass/vendor/bourbon/css3/_flex-box.scss +327 -0
  56. data/docs/_sass/vendor/bourbon/css3/_font-face.scss +29 -0
  57. data/docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss +6 -0
  58. data/docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss +12 -0
  59. data/docs/_sass/vendor/bourbon/css3/_hyphens.scss +6 -0
  60. data/docs/_sass/vendor/bourbon/css3/_image-rendering.scss +15 -0
  61. data/docs/_sass/vendor/bourbon/css3/_keyframes.scss +38 -0
  62. data/docs/_sass/vendor/bourbon/css3/_linear-gradient.scss +40 -0
  63. data/docs/_sass/vendor/bourbon/css3/_perspective.scss +12 -0
  64. data/docs/_sass/vendor/bourbon/css3/_placeholder.scss +10 -0
  65. data/docs/_sass/vendor/bourbon/css3/_radial-gradient.scss +40 -0
  66. data/docs/_sass/vendor/bourbon/css3/_selection.scss +44 -0
  67. data/docs/_sass/vendor/bourbon/css3/_text-decoration.scss +27 -0
  68. data/docs/_sass/vendor/bourbon/css3/_transform.scss +21 -0
  69. data/docs/_sass/vendor/bourbon/css3/_transition.scss +81 -0
  70. data/docs/_sass/vendor/bourbon/css3/_user-select.scss +5 -0
  71. data/docs/_sass/vendor/bourbon/functions/_assign-inputs.scss +16 -0
  72. data/docs/_sass/vendor/bourbon/functions/_contains-falsy.scss +25 -0
  73. data/docs/_sass/vendor/bourbon/functions/_contains.scss +31 -0
  74. data/docs/_sass/vendor/bourbon/functions/_is-length.scss +16 -0
  75. data/docs/_sass/vendor/bourbon/functions/_is-light.scss +26 -0
  76. data/docs/_sass/vendor/bourbon/functions/_is-number.scss +16 -0
  77. data/docs/_sass/vendor/bourbon/functions/_is-size.scss +23 -0
  78. data/docs/_sass/vendor/bourbon/functions/_modular-scale.scss +74 -0
  79. data/docs/_sass/vendor/bourbon/functions/_px-to-em.scss +24 -0
  80. data/docs/_sass/vendor/bourbon/functions/_px-to-rem.scss +26 -0
  81. data/docs/_sass/vendor/bourbon/functions/_shade.scss +24 -0
  82. data/docs/_sass/vendor/bourbon/functions/_strip-units.scss +22 -0
  83. data/docs/_sass/vendor/bourbon/functions/_tint.scss +24 -0
  84. data/docs/_sass/vendor/bourbon/functions/_transition-property-name.scss +37 -0
  85. data/docs/_sass/vendor/bourbon/functions/_unpack.scss +32 -0
  86. data/docs/_sass/vendor/bourbon/helpers/_convert-units.scss +26 -0
  87. data/docs/_sass/vendor/bourbon/helpers/_directional-values.scss +108 -0
  88. data/docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss +53 -0
  89. data/docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss +24 -0
  90. data/docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss +35 -0
  91. data/docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss +51 -0
  92. data/docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss +77 -0
  93. data/docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss +41 -0
  94. data/docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss +74 -0
  95. data/docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss +55 -0
  96. data/docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss +28 -0
  97. data/docs/_sass/vendor/bourbon/helpers/_render-gradients.scss +31 -0
  98. data/docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss +15 -0
  99. data/docs/_sass/vendor/bourbon/helpers/_str-to-num.scss +55 -0
  100. data/docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss +7 -0
  101. data/docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss +8 -0
  102. data/docs/_sass/vendor/bourbon/settings/_prefixer.scss +9 -0
  103. data/docs/_sass/vendor/bourbon/settings/_px-to-em.scss +1 -0
  104. data/docs/_sass/vendor/neat/_neat-helpers.scss +11 -0
  105. data/docs/_sass/vendor/neat/_neat.scss +23 -0
  106. data/docs/_sass/vendor/neat/functions/_new-breakpoint.scss +49 -0
  107. data/docs/_sass/vendor/neat/functions/_private.scss +114 -0
  108. data/docs/_sass/vendor/neat/grid/_box-sizing.scss +15 -0
  109. data/docs/_sass/vendor/neat/grid/_direction-context.scss +33 -0
  110. data/docs/_sass/vendor/neat/grid/_display-context.scss +28 -0
  111. data/docs/_sass/vendor/neat/grid/_fill-parent.scss +22 -0
  112. data/docs/_sass/vendor/neat/grid/_media.scss +92 -0
  113. data/docs/_sass/vendor/neat/grid/_omega.scss +87 -0
  114. data/docs/_sass/vendor/neat/grid/_outer-container.scss +34 -0
  115. data/docs/_sass/vendor/neat/grid/_pad.scss +25 -0
  116. data/docs/_sass/vendor/neat/grid/_private.scss +35 -0
  117. data/docs/_sass/vendor/neat/grid/_row.scss +52 -0
  118. data/docs/_sass/vendor/neat/grid/_shift.scss +50 -0
  119. data/docs/_sass/vendor/neat/grid/_span-columns.scss +94 -0
  120. data/docs/_sass/vendor/neat/grid/_to-deprecate.scss +97 -0
  121. data/docs/_sass/vendor/neat/grid/_visual-grid.scss +42 -0
  122. data/docs/_sass/vendor/neat/mixins/_clearfix.scss +25 -0
  123. data/docs/_sass/vendor/neat/settings/_disable-warnings.scss +13 -0
  124. data/docs/_sass/vendor/neat/settings/_grid.scss +51 -0
  125. data/docs/_sass/vendor/neat/settings/_visual-grid.scss +27 -0
  126. data/docs/_sass/vendor/normalize-3.0.2.scss +427 -0
  127. data/docs/_sass/vendor/pygments.scss +356 -0
  128. data/docs/automating_browsers/capybara.md +70 -0
  129. data/docs/css/screen.scss +7 -0
  130. data/docs/guides/callbacks.md +45 -0
  131. data/docs/guides/cli.md +52 -0
  132. data/docs/guides/configuration.md +184 -0
  133. data/docs/guides/error_handling.md +46 -0
  134. data/docs/guides/frontiers.md +93 -0
  135. data/docs/guides/halting.md +23 -0
  136. data/docs/guides/job_queues.md +26 -0
  137. data/docs/guides/locals.md +36 -0
  138. data/docs/guides/logging.md +22 -0
  139. data/docs/guides/page_objects.md +67 -0
  140. data/docs/guides/peeking.md +46 -0
  141. data/docs/guides/selenium_capybara.md +100 -0
  142. data/docs/guides/tutorial.md +452 -0
  143. data/docs/index.md +82 -0
  144. data/docs/js/navigation.js +11 -0
  145. data/docs/misc/contributing.md +20 -0
  146. data/docs/misc/testing.md +11 -0
  147. data/docs/recipes/authentication.md +23 -0
  148. data/docs/recipes/csv.md +29 -0
  149. data/docs/recipes/javascript.md +20 -0
  150. data/docs/recipes/multiple_uris.md +18 -0
  151. data/docs/recipes/screenshots.md +20 -0
  152. data/docs/routing/custom_rules.md +16 -0
  153. data/docs/routing/filetypes_rules.md +21 -0
  154. data/docs/routing/host_rules.md +24 -0
  155. data/docs/routing/path_rules.md +33 -0
  156. data/docs/routing/protocol_rules.md +17 -0
  157. data/docs/routing/query_rules.md +69 -0
  158. data/docs/routing/routes.md +96 -0
  159. data/docs/routing/uri_rules.md +18 -0
  160. data/examples/collect_github_issues.rb +65 -0
  161. data/examples/find_foobar_on_wikipedia.rb +23 -0
  162. data/lib/wayfarer/configuration.rb +86 -0
  163. data/lib/wayfarer/crawl.rb +79 -0
  164. data/lib/wayfarer/crawl_observer.rb +103 -0
  165. data/lib/wayfarer/dispatcher.rb +104 -0
  166. data/lib/wayfarer/finders.rb +61 -0
  167. data/lib/wayfarer/frontiers/frontier.rb +79 -0
  168. data/lib/wayfarer/frontiers/memory_bloomfilter.rb +32 -0
  169. data/lib/wayfarer/frontiers/memory_frontier.rb +76 -0
  170. data/lib/wayfarer/frontiers/memory_trie_frontier.rb +39 -0
  171. data/lib/wayfarer/frontiers/normalize_uris.rb +48 -0
  172. data/lib/wayfarer/frontiers/redis_bloomfilter.rb +34 -0
  173. data/lib/wayfarer/frontiers/redis_frontier.rb +83 -0
  174. data/lib/wayfarer/http_adapters/adapter_pool.rb +62 -0
  175. data/lib/wayfarer/http_adapters/net_http_adapter.rb +77 -0
  176. data/lib/wayfarer/http_adapters/selenium_adapter.rb +80 -0
  177. data/lib/wayfarer/job.rb +211 -0
  178. data/lib/wayfarer/locals.rb +40 -0
  179. data/lib/wayfarer/page.rb +94 -0
  180. data/lib/wayfarer/parsers/json_parser.rb +20 -0
  181. data/lib/wayfarer/parsers/xml_parser.rb +27 -0
  182. data/lib/wayfarer/processor.rb +103 -0
  183. data/lib/wayfarer/routing/custom_rule.rb +21 -0
  184. data/lib/wayfarer/routing/filetypes_rule.rb +20 -0
  185. data/lib/wayfarer/routing/host_rule.rb +19 -0
  186. data/lib/wayfarer/routing/path_rule.rb +54 -0
  187. data/lib/wayfarer/routing/protocol_rule.rb +21 -0
  188. data/lib/wayfarer/routing/query_rule.rb +59 -0
  189. data/lib/wayfarer/routing/router.rb +71 -0
  190. data/lib/wayfarer/routing/rule.rb +114 -0
  191. data/lib/wayfarer/routing/uri_rule.rb +21 -0
  192. data/lib/wayfarer.rb +68 -0
  193. data/spec/configuration_spec.rb +26 -0
  194. data/spec/crawl_spec.rb +48 -0
  195. data/spec/finders_spec.rb +49 -0
  196. data/spec/frontiers/memory_bloomfilter_spec.rb +6 -0
  197. data/spec/frontiers/memory_frontier_spec.rb +6 -0
  198. data/spec/frontiers/memory_trie_frontier_spec.rb +6 -0
  199. data/spec/frontiers/normalize_uris_spec.rb +59 -0
  200. data/spec/frontiers/redis_bloomfilter_spec.rb +6 -0
  201. data/spec/frontiers/redis_frontier_spec.rb +6 -0
  202. data/spec/http_adapters/adapter_pool_spec.rb +33 -0
  203. data/spec/http_adapters/net_http_adapter_spec.rb +83 -0
  204. data/spec/http_adapters/selenium_adapter_spec.rb +53 -0
  205. data/spec/integration/callbacks_spec.rb +42 -0
  206. data/spec/integration/locals_spec.rb +106 -0
  207. data/spec/integration/peeking_spec.rb +61 -0
  208. data/spec/job_spec.rb +122 -0
  209. data/spec/page_spec.rb +38 -0
  210. data/spec/parsers/json_parser_spec.rb +30 -0
  211. data/spec/parsers/xml_parser_spec.rb +24 -0
  212. data/spec/processor_spec.rb +31 -0
  213. data/spec/routing/custom_rule_spec.rb +26 -0
  214. data/spec/routing/filetypes_rule_spec.rb +40 -0
  215. data/spec/routing/host_rule_spec.rb +48 -0
  216. data/spec/routing/path_rule_spec.rb +66 -0
  217. data/spec/routing/protocol_rule_spec.rb +26 -0
  218. data/spec/routing/query_rule_spec.rb +124 -0
  219. data/spec/routing/router_spec.rb +67 -0
  220. data/spec/routing/rule_spec.rb +251 -0
  221. data/spec/routing/uri_rule_spec.rb +24 -0
  222. data/spec/shared/frontier.rb +96 -0
  223. data/spec/spec_helpers.rb +62 -0
  224. data/spec/wayfarer_spec.rb +24 -0
  225. data/support/static/finders.html +38 -0
  226. data/support/static/graph/details/a.html +10 -0
  227. data/support/static/graph/details/b.html +10 -0
  228. data/support/static/graph/index.html +20 -0
  229. data/support/static/json/dummy.json +13 -0
  230. data/support/static/links/links.html +28 -0
  231. data/support/static/xml/dummy.xml +120 -0
  232. data/support/test_app.rb +45 -0
  233. data/wayfarer-jruby.gemspec +49 -0
  234. data/wayfarer.gemspec +53 -0
  235. metadata +697 -0
metadata ADDED
@@ -0,0 +1,697 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: wayfarer
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.3
5
+ platform: ruby
6
+ authors:
7
+ - Dominic Bauer
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-05-31 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: connection_pool
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2.2'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '2.2'
27
+ - !ruby/object:Gem::Dependency
28
+ name: nokogiri
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.6'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.6'
41
+ - !ruby/object:Gem::Dependency
42
+ name: selenium-webdriver
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.4'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.4'
55
+ - !ruby/object:Gem::Dependency
56
+ name: selenium-emulated_features
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - '='
60
+ - !ruby/object:Gem::Version
61
+ version: 2.0.1
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '='
67
+ - !ruby/object:Gem::Version
68
+ version: 2.0.1
69
+ - !ruby/object:Gem::Dependency
70
+ name: net-http-persistent
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '2.9'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '2.9'
83
+ - !ruby/object:Gem::Dependency
84
+ name: mime-types
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '2.6'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '2.6'
97
+ - !ruby/object:Gem::Dependency
98
+ name: pismo
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '0.7'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '0.7'
111
+ - !ruby/object:Gem::Dependency
112
+ name: mustermann
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.4'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.4'
125
+ - !ruby/object:Gem::Dependency
126
+ name: activejob
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '4.2'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '4.2'
139
+ - !ruby/object:Gem::Dependency
140
+ name: thor
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - "~>"
144
+ - !ruby/object:Gem::Version
145
+ version: '0.19'
146
+ type: :runtime
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - "~>"
151
+ - !ruby/object:Gem::Version
152
+ version: '0.19'
153
+ - !ruby/object:Gem::Dependency
154
+ name: activesupport
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - "~>"
158
+ - !ruby/object:Gem::Version
159
+ version: '4.2'
160
+ type: :runtime
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - "~>"
165
+ - !ruby/object:Gem::Version
166
+ version: '4.2'
167
+ - !ruby/object:Gem::Dependency
168
+ name: capybara
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: '2.5'
174
+ type: :runtime
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "~>"
179
+ - !ruby/object:Gem::Version
180
+ version: '2.5'
181
+ - !ruby/object:Gem::Dependency
182
+ name: chronic
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '0.10'
188
+ type: :runtime
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '0.10'
195
+ - !ruby/object:Gem::Dependency
196
+ name: redis
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: '3.2'
202
+ type: :runtime
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: '3.2'
209
+ - !ruby/object:Gem::Dependency
210
+ name: concurrent-ruby
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - "~>"
214
+ - !ruby/object:Gem::Version
215
+ version: '1.0'
216
+ type: :runtime
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - "~>"
221
+ - !ruby/object:Gem::Version
222
+ version: '1.0'
223
+ - !ruby/object:Gem::Dependency
224
+ name: hooks
225
+ requirement: !ruby/object:Gem::Requirement
226
+ requirements:
227
+ - - "~>"
228
+ - !ruby/object:Gem::Version
229
+ version: '0.4'
230
+ type: :runtime
231
+ prerelease: false
232
+ version_requirements: !ruby/object:Gem::Requirement
233
+ requirements:
234
+ - - "~>"
235
+ - !ruby/object:Gem::Version
236
+ version: '0.4'
237
+ - !ruby/object:Gem::Dependency
238
+ name: bloomfilter-rb
239
+ requirement: !ruby/object:Gem::Requirement
240
+ requirements:
241
+ - - "~>"
242
+ - !ruby/object:Gem::Version
243
+ version: '2.1'
244
+ type: :runtime
245
+ prerelease: false
246
+ version_requirements: !ruby/object:Gem::Requirement
247
+ requirements:
248
+ - - "~>"
249
+ - !ruby/object:Gem::Version
250
+ version: '2.1'
251
+ - !ruby/object:Gem::Dependency
252
+ name: oj
253
+ requirement: !ruby/object:Gem::Requirement
254
+ requirements:
255
+ - - "~>"
256
+ - !ruby/object:Gem::Version
257
+ version: '2.12'
258
+ type: :runtime
259
+ prerelease: false
260
+ version_requirements: !ruby/object:Gem::Requirement
261
+ requirements:
262
+ - - "~>"
263
+ - !ruby/object:Gem::Version
264
+ version: '2.12'
265
+ - !ruby/object:Gem::Dependency
266
+ name: fast_trie
267
+ requirement: !ruby/object:Gem::Requirement
268
+ requirements:
269
+ - - "~>"
270
+ - !ruby/object:Gem::Version
271
+ version: '0.5'
272
+ type: :runtime
273
+ prerelease: false
274
+ version_requirements: !ruby/object:Gem::Requirement
275
+ requirements:
276
+ - - "~>"
277
+ - !ruby/object:Gem::Version
278
+ version: '0.5'
279
+ - !ruby/object:Gem::Dependency
280
+ name: parallel
281
+ requirement: !ruby/object:Gem::Requirement
282
+ requirements:
283
+ - - "~>"
284
+ - !ruby/object:Gem::Version
285
+ version: '1.11'
286
+ type: :runtime
287
+ prerelease: false
288
+ version_requirements: !ruby/object:Gem::Requirement
289
+ requirements:
290
+ - - "~>"
291
+ - !ruby/object:Gem::Version
292
+ version: '1.11'
293
+ - !ruby/object:Gem::Dependency
294
+ name: normalize_url
295
+ requirement: !ruby/object:Gem::Requirement
296
+ requirements:
297
+ - - "~>"
298
+ - !ruby/object:Gem::Version
299
+ version: 0.0.6
300
+ type: :runtime
301
+ prerelease: false
302
+ version_requirements: !ruby/object:Gem::Requirement
303
+ requirements:
304
+ - - "~>"
305
+ - !ruby/object:Gem::Version
306
+ version: 0.0.6
307
+ - !ruby/object:Gem::Dependency
308
+ name: pry
309
+ requirement: !ruby/object:Gem::Requirement
310
+ requirements:
311
+ - - "~>"
312
+ - !ruby/object:Gem::Version
313
+ version: '0.10'
314
+ type: :development
315
+ prerelease: false
316
+ version_requirements: !ruby/object:Gem::Requirement
317
+ requirements:
318
+ - - "~>"
319
+ - !ruby/object:Gem::Version
320
+ version: '0.10'
321
+ - !ruby/object:Gem::Dependency
322
+ name: rake
323
+ requirement: !ruby/object:Gem::Requirement
324
+ requirements:
325
+ - - "~>"
326
+ - !ruby/object:Gem::Version
327
+ version: '10.4'
328
+ type: :development
329
+ prerelease: false
330
+ version_requirements: !ruby/object:Gem::Requirement
331
+ requirements:
332
+ - - "~>"
333
+ - !ruby/object:Gem::Version
334
+ version: '10.4'
335
+ - !ruby/object:Gem::Dependency
336
+ name: rspec
337
+ requirement: !ruby/object:Gem::Requirement
338
+ requirements:
339
+ - - "~>"
340
+ - !ruby/object:Gem::Version
341
+ version: '3.3'
342
+ type: :development
343
+ prerelease: false
344
+ version_requirements: !ruby/object:Gem::Requirement
345
+ requirements:
346
+ - - "~>"
347
+ - !ruby/object:Gem::Version
348
+ version: '3.3'
349
+ - !ruby/object:Gem::Dependency
350
+ name: sinatra
351
+ requirement: !ruby/object:Gem::Requirement
352
+ requirements:
353
+ - - "~>"
354
+ - !ruby/object:Gem::Version
355
+ version: '1.4'
356
+ type: :development
357
+ prerelease: false
358
+ version_requirements: !ruby/object:Gem::Requirement
359
+ requirements:
360
+ - - "~>"
361
+ - !ruby/object:Gem::Version
362
+ version: '1.4'
363
+ - !ruby/object:Gem::Dependency
364
+ name: rubocop
365
+ requirement: !ruby/object:Gem::Requirement
366
+ requirements:
367
+ - - "~>"
368
+ - !ruby/object:Gem::Version
369
+ version: '0.34'
370
+ type: :development
371
+ prerelease: false
372
+ version_requirements: !ruby/object:Gem::Requirement
373
+ requirements:
374
+ - - "~>"
375
+ - !ruby/object:Gem::Version
376
+ version: '0.34'
377
+ - !ruby/object:Gem::Dependency
378
+ name: yard
379
+ requirement: !ruby/object:Gem::Requirement
380
+ requirements:
381
+ - - "~>"
382
+ - !ruby/object:Gem::Version
383
+ version: '0.9'
384
+ type: :development
385
+ prerelease: false
386
+ version_requirements: !ruby/object:Gem::Requirement
387
+ requirements:
388
+ - - "~>"
389
+ - !ruby/object:Gem::Version
390
+ version: '0.9'
391
+ - !ruby/object:Gem::Dependency
392
+ name: bourbon
393
+ requirement: !ruby/object:Gem::Requirement
394
+ requirements:
395
+ - - "~>"
396
+ - !ruby/object:Gem::Version
397
+ version: '4.3'
398
+ type: :development
399
+ prerelease: false
400
+ version_requirements: !ruby/object:Gem::Requirement
401
+ requirements:
402
+ - - "~>"
403
+ - !ruby/object:Gem::Version
404
+ version: '4.3'
405
+ - !ruby/object:Gem::Dependency
406
+ name: neat
407
+ requirement: !ruby/object:Gem::Requirement
408
+ requirements:
409
+ - - "~>"
410
+ - !ruby/object:Gem::Version
411
+ version: '1.8'
412
+ type: :development
413
+ prerelease: false
414
+ version_requirements: !ruby/object:Gem::Requirement
415
+ requirements:
416
+ - - "~>"
417
+ - !ruby/object:Gem::Version
418
+ version: '1.8'
419
+ - !ruby/object:Gem::Dependency
420
+ name: faker
421
+ requirement: !ruby/object:Gem::Requirement
422
+ requirements:
423
+ - - "~>"
424
+ - !ruby/object:Gem::Version
425
+ version: '1.7'
426
+ type: :development
427
+ prerelease: false
428
+ version_requirements: !ruby/object:Gem::Requirement
429
+ requirements:
430
+ - - "~>"
431
+ - !ruby/object:Gem::Version
432
+ version: '1.7'
433
+ description: Versatile web crawling with Ruby
434
+ email: bauerdominic@gmail.com
435
+ executables:
436
+ - wayfarer
437
+ extensions: []
438
+ extra_rdoc_files: []
439
+ files:
440
+ - ".gitignore"
441
+ - ".rbenv-gemsets"
442
+ - ".rspec"
443
+ - ".rubocop.yml"
444
+ - ".ruby-version"
445
+ - ".travis.yml"
446
+ - ".yardopts"
447
+ - Changelog.md
448
+ - Gemfile
449
+ - LICENSE
450
+ - README.md
451
+ - Rakefile
452
+ - benchmark/frontiers.rb
453
+ - bin/wayfarer
454
+ - docs/.gitignore
455
+ - docs/_config.yml
456
+ - docs/_includes/base.html
457
+ - docs/_includes/head.html
458
+ - docs/_includes/navigation.html
459
+ - docs/_layouts/default.html
460
+ - docs/_sass/base.scss
461
+ - docs/_sass/variables.scss
462
+ - docs/_sass/vendor/bourbon/_bourbon-deprecate.scss
463
+ - docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss
464
+ - docs/_sass/vendor/bourbon/_bourbon.scss
465
+ - docs/_sass/vendor/bourbon/addons/_border-color.scss
466
+ - docs/_sass/vendor/bourbon/addons/_border-radius.scss
467
+ - docs/_sass/vendor/bourbon/addons/_border-style.scss
468
+ - docs/_sass/vendor/bourbon/addons/_border-width.scss
469
+ - docs/_sass/vendor/bourbon/addons/_buttons.scss
470
+ - docs/_sass/vendor/bourbon/addons/_clearfix.scss
471
+ - docs/_sass/vendor/bourbon/addons/_ellipsis.scss
472
+ - docs/_sass/vendor/bourbon/addons/_font-stacks.scss
473
+ - docs/_sass/vendor/bourbon/addons/_hide-text.scss
474
+ - docs/_sass/vendor/bourbon/addons/_margin.scss
475
+ - docs/_sass/vendor/bourbon/addons/_padding.scss
476
+ - docs/_sass/vendor/bourbon/addons/_position.scss
477
+ - docs/_sass/vendor/bourbon/addons/_prefixer.scss
478
+ - docs/_sass/vendor/bourbon/addons/_retina-image.scss
479
+ - docs/_sass/vendor/bourbon/addons/_size.scss
480
+ - docs/_sass/vendor/bourbon/addons/_text-inputs.scss
481
+ - docs/_sass/vendor/bourbon/addons/_timing-functions.scss
482
+ - docs/_sass/vendor/bourbon/addons/_triangle.scss
483
+ - docs/_sass/vendor/bourbon/addons/_word-wrap.scss
484
+ - docs/_sass/vendor/bourbon/css3/_animation.scss
485
+ - docs/_sass/vendor/bourbon/css3/_appearance.scss
486
+ - docs/_sass/vendor/bourbon/css3/_backface-visibility.scss
487
+ - docs/_sass/vendor/bourbon/css3/_background-image.scss
488
+ - docs/_sass/vendor/bourbon/css3/_background.scss
489
+ - docs/_sass/vendor/bourbon/css3/_border-image.scss
490
+ - docs/_sass/vendor/bourbon/css3/_calc.scss
491
+ - docs/_sass/vendor/bourbon/css3/_columns.scss
492
+ - docs/_sass/vendor/bourbon/css3/_filter.scss
493
+ - docs/_sass/vendor/bourbon/css3/_flex-box.scss
494
+ - docs/_sass/vendor/bourbon/css3/_font-face.scss
495
+ - docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss
496
+ - docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss
497
+ - docs/_sass/vendor/bourbon/css3/_hyphens.scss
498
+ - docs/_sass/vendor/bourbon/css3/_image-rendering.scss
499
+ - docs/_sass/vendor/bourbon/css3/_keyframes.scss
500
+ - docs/_sass/vendor/bourbon/css3/_linear-gradient.scss
501
+ - docs/_sass/vendor/bourbon/css3/_perspective.scss
502
+ - docs/_sass/vendor/bourbon/css3/_placeholder.scss
503
+ - docs/_sass/vendor/bourbon/css3/_radial-gradient.scss
504
+ - docs/_sass/vendor/bourbon/css3/_selection.scss
505
+ - docs/_sass/vendor/bourbon/css3/_text-decoration.scss
506
+ - docs/_sass/vendor/bourbon/css3/_transform.scss
507
+ - docs/_sass/vendor/bourbon/css3/_transition.scss
508
+ - docs/_sass/vendor/bourbon/css3/_user-select.scss
509
+ - docs/_sass/vendor/bourbon/functions/_assign-inputs.scss
510
+ - docs/_sass/vendor/bourbon/functions/_contains-falsy.scss
511
+ - docs/_sass/vendor/bourbon/functions/_contains.scss
512
+ - docs/_sass/vendor/bourbon/functions/_is-length.scss
513
+ - docs/_sass/vendor/bourbon/functions/_is-light.scss
514
+ - docs/_sass/vendor/bourbon/functions/_is-number.scss
515
+ - docs/_sass/vendor/bourbon/functions/_is-size.scss
516
+ - docs/_sass/vendor/bourbon/functions/_modular-scale.scss
517
+ - docs/_sass/vendor/bourbon/functions/_px-to-em.scss
518
+ - docs/_sass/vendor/bourbon/functions/_px-to-rem.scss
519
+ - docs/_sass/vendor/bourbon/functions/_shade.scss
520
+ - docs/_sass/vendor/bourbon/functions/_strip-units.scss
521
+ - docs/_sass/vendor/bourbon/functions/_tint.scss
522
+ - docs/_sass/vendor/bourbon/functions/_transition-property-name.scss
523
+ - docs/_sass/vendor/bourbon/functions/_unpack.scss
524
+ - docs/_sass/vendor/bourbon/helpers/_convert-units.scss
525
+ - docs/_sass/vendor/bourbon/helpers/_directional-values.scss
526
+ - docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss
527
+ - docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss
528
+ - docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss
529
+ - docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss
530
+ - docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss
531
+ - docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss
532
+ - docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss
533
+ - docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss
534
+ - docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss
535
+ - docs/_sass/vendor/bourbon/helpers/_render-gradients.scss
536
+ - docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss
537
+ - docs/_sass/vendor/bourbon/helpers/_str-to-num.scss
538
+ - docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss
539
+ - docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss
540
+ - docs/_sass/vendor/bourbon/settings/_prefixer.scss
541
+ - docs/_sass/vendor/bourbon/settings/_px-to-em.scss
542
+ - docs/_sass/vendor/neat/_neat-helpers.scss
543
+ - docs/_sass/vendor/neat/_neat.scss
544
+ - docs/_sass/vendor/neat/functions/_new-breakpoint.scss
545
+ - docs/_sass/vendor/neat/functions/_private.scss
546
+ - docs/_sass/vendor/neat/grid/_box-sizing.scss
547
+ - docs/_sass/vendor/neat/grid/_direction-context.scss
548
+ - docs/_sass/vendor/neat/grid/_display-context.scss
549
+ - docs/_sass/vendor/neat/grid/_fill-parent.scss
550
+ - docs/_sass/vendor/neat/grid/_media.scss
551
+ - docs/_sass/vendor/neat/grid/_omega.scss
552
+ - docs/_sass/vendor/neat/grid/_outer-container.scss
553
+ - docs/_sass/vendor/neat/grid/_pad.scss
554
+ - docs/_sass/vendor/neat/grid/_private.scss
555
+ - docs/_sass/vendor/neat/grid/_row.scss
556
+ - docs/_sass/vendor/neat/grid/_shift.scss
557
+ - docs/_sass/vendor/neat/grid/_span-columns.scss
558
+ - docs/_sass/vendor/neat/grid/_to-deprecate.scss
559
+ - docs/_sass/vendor/neat/grid/_visual-grid.scss
560
+ - docs/_sass/vendor/neat/mixins/_clearfix.scss
561
+ - docs/_sass/vendor/neat/settings/_disable-warnings.scss
562
+ - docs/_sass/vendor/neat/settings/_grid.scss
563
+ - docs/_sass/vendor/neat/settings/_visual-grid.scss
564
+ - docs/_sass/vendor/normalize-3.0.2.scss
565
+ - docs/_sass/vendor/pygments.scss
566
+ - docs/automating_browsers/capybara.md
567
+ - docs/css/screen.scss
568
+ - docs/guides/callbacks.md
569
+ - docs/guides/cli.md
570
+ - docs/guides/configuration.md
571
+ - docs/guides/error_handling.md
572
+ - docs/guides/frontiers.md
573
+ - docs/guides/halting.md
574
+ - docs/guides/job_queues.md
575
+ - docs/guides/locals.md
576
+ - docs/guides/logging.md
577
+ - docs/guides/page_objects.md
578
+ - docs/guides/peeking.md
579
+ - docs/guides/selenium_capybara.md
580
+ - docs/guides/tutorial.md
581
+ - docs/index.md
582
+ - docs/js/navigation.js
583
+ - docs/misc/contributing.md
584
+ - docs/misc/testing.md
585
+ - docs/recipes/authentication.md
586
+ - docs/recipes/csv.md
587
+ - docs/recipes/javascript.md
588
+ - docs/recipes/multiple_uris.md
589
+ - docs/recipes/screenshots.md
590
+ - docs/routing/custom_rules.md
591
+ - docs/routing/filetypes_rules.md
592
+ - docs/routing/host_rules.md
593
+ - docs/routing/path_rules.md
594
+ - docs/routing/protocol_rules.md
595
+ - docs/routing/query_rules.md
596
+ - docs/routing/routes.md
597
+ - docs/routing/uri_rules.md
598
+ - examples/collect_github_issues.rb
599
+ - examples/find_foobar_on_wikipedia.rb
600
+ - lib/wayfarer.rb
601
+ - lib/wayfarer/configuration.rb
602
+ - lib/wayfarer/crawl.rb
603
+ - lib/wayfarer/crawl_observer.rb
604
+ - lib/wayfarer/dispatcher.rb
605
+ - lib/wayfarer/finders.rb
606
+ - lib/wayfarer/frontiers/frontier.rb
607
+ - lib/wayfarer/frontiers/memory_bloomfilter.rb
608
+ - lib/wayfarer/frontiers/memory_frontier.rb
609
+ - lib/wayfarer/frontiers/memory_trie_frontier.rb
610
+ - lib/wayfarer/frontiers/normalize_uris.rb
611
+ - lib/wayfarer/frontiers/redis_bloomfilter.rb
612
+ - lib/wayfarer/frontiers/redis_frontier.rb
613
+ - lib/wayfarer/http_adapters/adapter_pool.rb
614
+ - lib/wayfarer/http_adapters/net_http_adapter.rb
615
+ - lib/wayfarer/http_adapters/selenium_adapter.rb
616
+ - lib/wayfarer/job.rb
617
+ - lib/wayfarer/locals.rb
618
+ - lib/wayfarer/page.rb
619
+ - lib/wayfarer/parsers/json_parser.rb
620
+ - lib/wayfarer/parsers/xml_parser.rb
621
+ - lib/wayfarer/processor.rb
622
+ - lib/wayfarer/routing/custom_rule.rb
623
+ - lib/wayfarer/routing/filetypes_rule.rb
624
+ - lib/wayfarer/routing/host_rule.rb
625
+ - lib/wayfarer/routing/path_rule.rb
626
+ - lib/wayfarer/routing/protocol_rule.rb
627
+ - lib/wayfarer/routing/query_rule.rb
628
+ - lib/wayfarer/routing/router.rb
629
+ - lib/wayfarer/routing/rule.rb
630
+ - lib/wayfarer/routing/uri_rule.rb
631
+ - spec/configuration_spec.rb
632
+ - spec/crawl_spec.rb
633
+ - spec/finders_spec.rb
634
+ - spec/frontiers/memory_bloomfilter_spec.rb
635
+ - spec/frontiers/memory_frontier_spec.rb
636
+ - spec/frontiers/memory_trie_frontier_spec.rb
637
+ - spec/frontiers/normalize_uris_spec.rb
638
+ - spec/frontiers/redis_bloomfilter_spec.rb
639
+ - spec/frontiers/redis_frontier_spec.rb
640
+ - spec/http_adapters/adapter_pool_spec.rb
641
+ - spec/http_adapters/net_http_adapter_spec.rb
642
+ - spec/http_adapters/selenium_adapter_spec.rb
643
+ - spec/integration/callbacks_spec.rb
644
+ - spec/integration/locals_spec.rb
645
+ - spec/integration/peeking_spec.rb
646
+ - spec/job_spec.rb
647
+ - spec/page_spec.rb
648
+ - spec/parsers/json_parser_spec.rb
649
+ - spec/parsers/xml_parser_spec.rb
650
+ - spec/processor_spec.rb
651
+ - spec/routing/custom_rule_spec.rb
652
+ - spec/routing/filetypes_rule_spec.rb
653
+ - spec/routing/host_rule_spec.rb
654
+ - spec/routing/path_rule_spec.rb
655
+ - spec/routing/protocol_rule_spec.rb
656
+ - spec/routing/query_rule_spec.rb
657
+ - spec/routing/router_spec.rb
658
+ - spec/routing/rule_spec.rb
659
+ - spec/routing/uri_rule_spec.rb
660
+ - spec/shared/frontier.rb
661
+ - spec/spec_helpers.rb
662
+ - spec/wayfarer_spec.rb
663
+ - support/static/finders.html
664
+ - support/static/graph/details/a.html
665
+ - support/static/graph/details/b.html
666
+ - support/static/graph/index.html
667
+ - support/static/json/dummy.json
668
+ - support/static/links/links.html
669
+ - support/static/xml/dummy.xml
670
+ - support/test_app.rb
671
+ - wayfarer-jruby.gemspec
672
+ - wayfarer.gemspec
673
+ homepage: http://github.com/bauerd/wayfarer
674
+ licenses:
675
+ - MIT
676
+ metadata: {}
677
+ post_install_message:
678
+ rdoc_options: []
679
+ require_paths:
680
+ - lib
681
+ required_ruby_version: !ruby/object:Gem::Requirement
682
+ requirements:
683
+ - - ">="
684
+ - !ruby/object:Gem::Version
685
+ version: '0'
686
+ required_rubygems_version: !ruby/object:Gem::Requirement
687
+ requirements:
688
+ - - ">="
689
+ - !ruby/object:Gem::Version
690
+ version: '0'
691
+ requirements: []
692
+ rubyforge_project:
693
+ rubygems_version: 2.5.1
694
+ signing_key:
695
+ specification_version: 4
696
+ summary: Versatile web crawling with Ruby
697
+ test_files: []