wayfarer 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (235) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +8 -0
  3. data/.rbenv-gemsets +1 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +21 -0
  6. data/.ruby-version +1 -0
  7. data/.travis.yml +5 -0
  8. data/.yardopts +3 -0
  9. data/Changelog.md +10 -0
  10. data/Gemfile +11 -0
  11. data/LICENSE +19 -0
  12. data/README.md +21 -0
  13. data/Rakefile +114 -0
  14. data/benchmark/frontiers.rb +143 -0
  15. data/bin/wayfarer +116 -0
  16. data/docs/.gitignore +2 -0
  17. data/docs/_config.yml +15 -0
  18. data/docs/_includes/base.html +7 -0
  19. data/docs/_includes/head.html +10 -0
  20. data/docs/_includes/navigation.html +187 -0
  21. data/docs/_layouts/default.html +42 -0
  22. data/docs/_sass/base.scss +439 -0
  23. data/docs/_sass/variables.scss +24 -0
  24. data/docs/_sass/vendor/bourbon/_bourbon-deprecate.scss +19 -0
  25. data/docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss +425 -0
  26. data/docs/_sass/vendor/bourbon/_bourbon.scss +90 -0
  27. data/docs/_sass/vendor/bourbon/addons/_border-color.scss +29 -0
  28. data/docs/_sass/vendor/bourbon/addons/_border-radius.scss +48 -0
  29. data/docs/_sass/vendor/bourbon/addons/_border-style.scss +28 -0
  30. data/docs/_sass/vendor/bourbon/addons/_border-width.scss +28 -0
  31. data/docs/_sass/vendor/bourbon/addons/_buttons.scss +69 -0
  32. data/docs/_sass/vendor/bourbon/addons/_clearfix.scss +25 -0
  33. data/docs/_sass/vendor/bourbon/addons/_ellipsis.scss +30 -0
  34. data/docs/_sass/vendor/bourbon/addons/_font-stacks.scss +31 -0
  35. data/docs/_sass/vendor/bourbon/addons/_hide-text.scss +27 -0
  36. data/docs/_sass/vendor/bourbon/addons/_margin.scss +29 -0
  37. data/docs/_sass/vendor/bourbon/addons/_padding.scss +29 -0
  38. data/docs/_sass/vendor/bourbon/addons/_position.scss +51 -0
  39. data/docs/_sass/vendor/bourbon/addons/_prefixer.scss +66 -0
  40. data/docs/_sass/vendor/bourbon/addons/_retina-image.scss +27 -0
  41. data/docs/_sass/vendor/bourbon/addons/_size.scss +56 -0
  42. data/docs/_sass/vendor/bourbon/addons/_text-inputs.scss +118 -0
  43. data/docs/_sass/vendor/bourbon/addons/_timing-functions.scss +34 -0
  44. data/docs/_sass/vendor/bourbon/addons/_triangle.scss +63 -0
  45. data/docs/_sass/vendor/bourbon/addons/_word-wrap.scss +29 -0
  46. data/docs/_sass/vendor/bourbon/css3/_animation.scss +61 -0
  47. data/docs/_sass/vendor/bourbon/css3/_appearance.scss +5 -0
  48. data/docs/_sass/vendor/bourbon/css3/_backface-visibility.scss +5 -0
  49. data/docs/_sass/vendor/bourbon/css3/_background-image.scss +44 -0
  50. data/docs/_sass/vendor/bourbon/css3/_background.scss +57 -0
  51. data/docs/_sass/vendor/bourbon/css3/_border-image.scss +61 -0
  52. data/docs/_sass/vendor/bourbon/css3/_calc.scss +6 -0
  53. data/docs/_sass/vendor/bourbon/css3/_columns.scss +67 -0
  54. data/docs/_sass/vendor/bourbon/css3/_filter.scss +6 -0
  55. data/docs/_sass/vendor/bourbon/css3/_flex-box.scss +327 -0
  56. data/docs/_sass/vendor/bourbon/css3/_font-face.scss +29 -0
  57. data/docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss +6 -0
  58. data/docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss +12 -0
  59. data/docs/_sass/vendor/bourbon/css3/_hyphens.scss +6 -0
  60. data/docs/_sass/vendor/bourbon/css3/_image-rendering.scss +15 -0
  61. data/docs/_sass/vendor/bourbon/css3/_keyframes.scss +38 -0
  62. data/docs/_sass/vendor/bourbon/css3/_linear-gradient.scss +40 -0
  63. data/docs/_sass/vendor/bourbon/css3/_perspective.scss +12 -0
  64. data/docs/_sass/vendor/bourbon/css3/_placeholder.scss +10 -0
  65. data/docs/_sass/vendor/bourbon/css3/_radial-gradient.scss +40 -0
  66. data/docs/_sass/vendor/bourbon/css3/_selection.scss +44 -0
  67. data/docs/_sass/vendor/bourbon/css3/_text-decoration.scss +27 -0
  68. data/docs/_sass/vendor/bourbon/css3/_transform.scss +21 -0
  69. data/docs/_sass/vendor/bourbon/css3/_transition.scss +81 -0
  70. data/docs/_sass/vendor/bourbon/css3/_user-select.scss +5 -0
  71. data/docs/_sass/vendor/bourbon/functions/_assign-inputs.scss +16 -0
  72. data/docs/_sass/vendor/bourbon/functions/_contains-falsy.scss +25 -0
  73. data/docs/_sass/vendor/bourbon/functions/_contains.scss +31 -0
  74. data/docs/_sass/vendor/bourbon/functions/_is-length.scss +16 -0
  75. data/docs/_sass/vendor/bourbon/functions/_is-light.scss +26 -0
  76. data/docs/_sass/vendor/bourbon/functions/_is-number.scss +16 -0
  77. data/docs/_sass/vendor/bourbon/functions/_is-size.scss +23 -0
  78. data/docs/_sass/vendor/bourbon/functions/_modular-scale.scss +74 -0
  79. data/docs/_sass/vendor/bourbon/functions/_px-to-em.scss +24 -0
  80. data/docs/_sass/vendor/bourbon/functions/_px-to-rem.scss +26 -0
  81. data/docs/_sass/vendor/bourbon/functions/_shade.scss +24 -0
  82. data/docs/_sass/vendor/bourbon/functions/_strip-units.scss +22 -0
  83. data/docs/_sass/vendor/bourbon/functions/_tint.scss +24 -0
  84. data/docs/_sass/vendor/bourbon/functions/_transition-property-name.scss +37 -0
  85. data/docs/_sass/vendor/bourbon/functions/_unpack.scss +32 -0
  86. data/docs/_sass/vendor/bourbon/helpers/_convert-units.scss +26 -0
  87. data/docs/_sass/vendor/bourbon/helpers/_directional-values.scss +108 -0
  88. data/docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss +53 -0
  89. data/docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss +24 -0
  90. data/docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss +35 -0
  91. data/docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss +51 -0
  92. data/docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss +77 -0
  93. data/docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss +41 -0
  94. data/docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss +74 -0
  95. data/docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss +55 -0
  96. data/docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss +28 -0
  97. data/docs/_sass/vendor/bourbon/helpers/_render-gradients.scss +31 -0
  98. data/docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss +15 -0
  99. data/docs/_sass/vendor/bourbon/helpers/_str-to-num.scss +55 -0
  100. data/docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss +7 -0
  101. data/docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss +8 -0
  102. data/docs/_sass/vendor/bourbon/settings/_prefixer.scss +9 -0
  103. data/docs/_sass/vendor/bourbon/settings/_px-to-em.scss +1 -0
  104. data/docs/_sass/vendor/neat/_neat-helpers.scss +11 -0
  105. data/docs/_sass/vendor/neat/_neat.scss +23 -0
  106. data/docs/_sass/vendor/neat/functions/_new-breakpoint.scss +49 -0
  107. data/docs/_sass/vendor/neat/functions/_private.scss +114 -0
  108. data/docs/_sass/vendor/neat/grid/_box-sizing.scss +15 -0
  109. data/docs/_sass/vendor/neat/grid/_direction-context.scss +33 -0
  110. data/docs/_sass/vendor/neat/grid/_display-context.scss +28 -0
  111. data/docs/_sass/vendor/neat/grid/_fill-parent.scss +22 -0
  112. data/docs/_sass/vendor/neat/grid/_media.scss +92 -0
  113. data/docs/_sass/vendor/neat/grid/_omega.scss +87 -0
  114. data/docs/_sass/vendor/neat/grid/_outer-container.scss +34 -0
  115. data/docs/_sass/vendor/neat/grid/_pad.scss +25 -0
  116. data/docs/_sass/vendor/neat/grid/_private.scss +35 -0
  117. data/docs/_sass/vendor/neat/grid/_row.scss +52 -0
  118. data/docs/_sass/vendor/neat/grid/_shift.scss +50 -0
  119. data/docs/_sass/vendor/neat/grid/_span-columns.scss +94 -0
  120. data/docs/_sass/vendor/neat/grid/_to-deprecate.scss +97 -0
  121. data/docs/_sass/vendor/neat/grid/_visual-grid.scss +42 -0
  122. data/docs/_sass/vendor/neat/mixins/_clearfix.scss +25 -0
  123. data/docs/_sass/vendor/neat/settings/_disable-warnings.scss +13 -0
  124. data/docs/_sass/vendor/neat/settings/_grid.scss +51 -0
  125. data/docs/_sass/vendor/neat/settings/_visual-grid.scss +27 -0
  126. data/docs/_sass/vendor/normalize-3.0.2.scss +427 -0
  127. data/docs/_sass/vendor/pygments.scss +356 -0
  128. data/docs/automating_browsers/capybara.md +70 -0
  129. data/docs/css/screen.scss +7 -0
  130. data/docs/guides/callbacks.md +45 -0
  131. data/docs/guides/cli.md +52 -0
  132. data/docs/guides/configuration.md +184 -0
  133. data/docs/guides/error_handling.md +46 -0
  134. data/docs/guides/frontiers.md +93 -0
  135. data/docs/guides/halting.md +23 -0
  136. data/docs/guides/job_queues.md +26 -0
  137. data/docs/guides/locals.md +36 -0
  138. data/docs/guides/logging.md +22 -0
  139. data/docs/guides/page_objects.md +67 -0
  140. data/docs/guides/peeking.md +46 -0
  141. data/docs/guides/selenium_capybara.md +100 -0
  142. data/docs/guides/tutorial.md +452 -0
  143. data/docs/index.md +82 -0
  144. data/docs/js/navigation.js +11 -0
  145. data/docs/misc/contributing.md +20 -0
  146. data/docs/misc/testing.md +11 -0
  147. data/docs/recipes/authentication.md +23 -0
  148. data/docs/recipes/csv.md +29 -0
  149. data/docs/recipes/javascript.md +20 -0
  150. data/docs/recipes/multiple_uris.md +18 -0
  151. data/docs/recipes/screenshots.md +20 -0
  152. data/docs/routing/custom_rules.md +16 -0
  153. data/docs/routing/filetypes_rules.md +21 -0
  154. data/docs/routing/host_rules.md +24 -0
  155. data/docs/routing/path_rules.md +33 -0
  156. data/docs/routing/protocol_rules.md +17 -0
  157. data/docs/routing/query_rules.md +69 -0
  158. data/docs/routing/routes.md +96 -0
  159. data/docs/routing/uri_rules.md +18 -0
  160. data/examples/collect_github_issues.rb +65 -0
  161. data/examples/find_foobar_on_wikipedia.rb +23 -0
  162. data/lib/wayfarer/configuration.rb +86 -0
  163. data/lib/wayfarer/crawl.rb +79 -0
  164. data/lib/wayfarer/crawl_observer.rb +103 -0
  165. data/lib/wayfarer/dispatcher.rb +104 -0
  166. data/lib/wayfarer/finders.rb +61 -0
  167. data/lib/wayfarer/frontiers/frontier.rb +79 -0
  168. data/lib/wayfarer/frontiers/memory_bloomfilter.rb +32 -0
  169. data/lib/wayfarer/frontiers/memory_frontier.rb +76 -0
  170. data/lib/wayfarer/frontiers/memory_trie_frontier.rb +39 -0
  171. data/lib/wayfarer/frontiers/normalize_uris.rb +48 -0
  172. data/lib/wayfarer/frontiers/redis_bloomfilter.rb +34 -0
  173. data/lib/wayfarer/frontiers/redis_frontier.rb +83 -0
  174. data/lib/wayfarer/http_adapters/adapter_pool.rb +62 -0
  175. data/lib/wayfarer/http_adapters/net_http_adapter.rb +77 -0
  176. data/lib/wayfarer/http_adapters/selenium_adapter.rb +80 -0
  177. data/lib/wayfarer/job.rb +211 -0
  178. data/lib/wayfarer/locals.rb +40 -0
  179. data/lib/wayfarer/page.rb +94 -0
  180. data/lib/wayfarer/parsers/json_parser.rb +20 -0
  181. data/lib/wayfarer/parsers/xml_parser.rb +27 -0
  182. data/lib/wayfarer/processor.rb +103 -0
  183. data/lib/wayfarer/routing/custom_rule.rb +21 -0
  184. data/lib/wayfarer/routing/filetypes_rule.rb +20 -0
  185. data/lib/wayfarer/routing/host_rule.rb +19 -0
  186. data/lib/wayfarer/routing/path_rule.rb +54 -0
  187. data/lib/wayfarer/routing/protocol_rule.rb +21 -0
  188. data/lib/wayfarer/routing/query_rule.rb +59 -0
  189. data/lib/wayfarer/routing/router.rb +71 -0
  190. data/lib/wayfarer/routing/rule.rb +114 -0
  191. data/lib/wayfarer/routing/uri_rule.rb +21 -0
  192. data/lib/wayfarer.rb +68 -0
  193. data/spec/configuration_spec.rb +26 -0
  194. data/spec/crawl_spec.rb +48 -0
  195. data/spec/finders_spec.rb +49 -0
  196. data/spec/frontiers/memory_bloomfilter_spec.rb +6 -0
  197. data/spec/frontiers/memory_frontier_spec.rb +6 -0
  198. data/spec/frontiers/memory_trie_frontier_spec.rb +6 -0
  199. data/spec/frontiers/normalize_uris_spec.rb +59 -0
  200. data/spec/frontiers/redis_bloomfilter_spec.rb +6 -0
  201. data/spec/frontiers/redis_frontier_spec.rb +6 -0
  202. data/spec/http_adapters/adapter_pool_spec.rb +33 -0
  203. data/spec/http_adapters/net_http_adapter_spec.rb +83 -0
  204. data/spec/http_adapters/selenium_adapter_spec.rb +53 -0
  205. data/spec/integration/callbacks_spec.rb +42 -0
  206. data/spec/integration/locals_spec.rb +106 -0
  207. data/spec/integration/peeking_spec.rb +61 -0
  208. data/spec/job_spec.rb +122 -0
  209. data/spec/page_spec.rb +38 -0
  210. data/spec/parsers/json_parser_spec.rb +30 -0
  211. data/spec/parsers/xml_parser_spec.rb +24 -0
  212. data/spec/processor_spec.rb +31 -0
  213. data/spec/routing/custom_rule_spec.rb +26 -0
  214. data/spec/routing/filetypes_rule_spec.rb +40 -0
  215. data/spec/routing/host_rule_spec.rb +48 -0
  216. data/spec/routing/path_rule_spec.rb +66 -0
  217. data/spec/routing/protocol_rule_spec.rb +26 -0
  218. data/spec/routing/query_rule_spec.rb +124 -0
  219. data/spec/routing/router_spec.rb +67 -0
  220. data/spec/routing/rule_spec.rb +251 -0
  221. data/spec/routing/uri_rule_spec.rb +24 -0
  222. data/spec/shared/frontier.rb +96 -0
  223. data/spec/spec_helpers.rb +62 -0
  224. data/spec/wayfarer_spec.rb +24 -0
  225. data/support/static/finders.html +38 -0
  226. data/support/static/graph/details/a.html +10 -0
  227. data/support/static/graph/details/b.html +10 -0
  228. data/support/static/graph/index.html +20 -0
  229. data/support/static/json/dummy.json +13 -0
  230. data/support/static/links/links.html +28 -0
  231. data/support/static/xml/dummy.xml +120 -0
  232. data/support/test_app.rb +45 -0
  233. data/wayfarer-jruby.gemspec +49 -0
  234. data/wayfarer.gemspec +53 -0
  235. metadata +697 -0
metadata ADDED
@@ -0,0 +1,697 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: wayfarer
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.3
5
+ platform: ruby
6
+ authors:
7
+ - Dominic Bauer
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-05-31 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: connection_pool
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2.2'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '2.2'
27
+ - !ruby/object:Gem::Dependency
28
+ name: nokogiri
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.6'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.6'
41
+ - !ruby/object:Gem::Dependency
42
+ name: selenium-webdriver
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.4'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.4'
55
+ - !ruby/object:Gem::Dependency
56
+ name: selenium-emulated_features
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - '='
60
+ - !ruby/object:Gem::Version
61
+ version: 2.0.1
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '='
67
+ - !ruby/object:Gem::Version
68
+ version: 2.0.1
69
+ - !ruby/object:Gem::Dependency
70
+ name: net-http-persistent
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '2.9'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '2.9'
83
+ - !ruby/object:Gem::Dependency
84
+ name: mime-types
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '2.6'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '2.6'
97
+ - !ruby/object:Gem::Dependency
98
+ name: pismo
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '0.7'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '0.7'
111
+ - !ruby/object:Gem::Dependency
112
+ name: mustermann
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.4'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.4'
125
+ - !ruby/object:Gem::Dependency
126
+ name: activejob
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '4.2'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '4.2'
139
+ - !ruby/object:Gem::Dependency
140
+ name: thor
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - "~>"
144
+ - !ruby/object:Gem::Version
145
+ version: '0.19'
146
+ type: :runtime
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - "~>"
151
+ - !ruby/object:Gem::Version
152
+ version: '0.19'
153
+ - !ruby/object:Gem::Dependency
154
+ name: activesupport
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - "~>"
158
+ - !ruby/object:Gem::Version
159
+ version: '4.2'
160
+ type: :runtime
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - "~>"
165
+ - !ruby/object:Gem::Version
166
+ version: '4.2'
167
+ - !ruby/object:Gem::Dependency
168
+ name: capybara
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: '2.5'
174
+ type: :runtime
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "~>"
179
+ - !ruby/object:Gem::Version
180
+ version: '2.5'
181
+ - !ruby/object:Gem::Dependency
182
+ name: chronic
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '0.10'
188
+ type: :runtime
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '0.10'
195
+ - !ruby/object:Gem::Dependency
196
+ name: redis
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: '3.2'
202
+ type: :runtime
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: '3.2'
209
+ - !ruby/object:Gem::Dependency
210
+ name: concurrent-ruby
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - "~>"
214
+ - !ruby/object:Gem::Version
215
+ version: '1.0'
216
+ type: :runtime
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - "~>"
221
+ - !ruby/object:Gem::Version
222
+ version: '1.0'
223
+ - !ruby/object:Gem::Dependency
224
+ name: hooks
225
+ requirement: !ruby/object:Gem::Requirement
226
+ requirements:
227
+ - - "~>"
228
+ - !ruby/object:Gem::Version
229
+ version: '0.4'
230
+ type: :runtime
231
+ prerelease: false
232
+ version_requirements: !ruby/object:Gem::Requirement
233
+ requirements:
234
+ - - "~>"
235
+ - !ruby/object:Gem::Version
236
+ version: '0.4'
237
+ - !ruby/object:Gem::Dependency
238
+ name: bloomfilter-rb
239
+ requirement: !ruby/object:Gem::Requirement
240
+ requirements:
241
+ - - "~>"
242
+ - !ruby/object:Gem::Version
243
+ version: '2.1'
244
+ type: :runtime
245
+ prerelease: false
246
+ version_requirements: !ruby/object:Gem::Requirement
247
+ requirements:
248
+ - - "~>"
249
+ - !ruby/object:Gem::Version
250
+ version: '2.1'
251
+ - !ruby/object:Gem::Dependency
252
+ name: oj
253
+ requirement: !ruby/object:Gem::Requirement
254
+ requirements:
255
+ - - "~>"
256
+ - !ruby/object:Gem::Version
257
+ version: '2.12'
258
+ type: :runtime
259
+ prerelease: false
260
+ version_requirements: !ruby/object:Gem::Requirement
261
+ requirements:
262
+ - - "~>"
263
+ - !ruby/object:Gem::Version
264
+ version: '2.12'
265
+ - !ruby/object:Gem::Dependency
266
+ name: fast_trie
267
+ requirement: !ruby/object:Gem::Requirement
268
+ requirements:
269
+ - - "~>"
270
+ - !ruby/object:Gem::Version
271
+ version: '0.5'
272
+ type: :runtime
273
+ prerelease: false
274
+ version_requirements: !ruby/object:Gem::Requirement
275
+ requirements:
276
+ - - "~>"
277
+ - !ruby/object:Gem::Version
278
+ version: '0.5'
279
+ - !ruby/object:Gem::Dependency
280
+ name: parallel
281
+ requirement: !ruby/object:Gem::Requirement
282
+ requirements:
283
+ - - "~>"
284
+ - !ruby/object:Gem::Version
285
+ version: '1.11'
286
+ type: :runtime
287
+ prerelease: false
288
+ version_requirements: !ruby/object:Gem::Requirement
289
+ requirements:
290
+ - - "~>"
291
+ - !ruby/object:Gem::Version
292
+ version: '1.11'
293
+ - !ruby/object:Gem::Dependency
294
+ name: normalize_url
295
+ requirement: !ruby/object:Gem::Requirement
296
+ requirements:
297
+ - - "~>"
298
+ - !ruby/object:Gem::Version
299
+ version: 0.0.6
300
+ type: :runtime
301
+ prerelease: false
302
+ version_requirements: !ruby/object:Gem::Requirement
303
+ requirements:
304
+ - - "~>"
305
+ - !ruby/object:Gem::Version
306
+ version: 0.0.6
307
+ - !ruby/object:Gem::Dependency
308
+ name: pry
309
+ requirement: !ruby/object:Gem::Requirement
310
+ requirements:
311
+ - - "~>"
312
+ - !ruby/object:Gem::Version
313
+ version: '0.10'
314
+ type: :development
315
+ prerelease: false
316
+ version_requirements: !ruby/object:Gem::Requirement
317
+ requirements:
318
+ - - "~>"
319
+ - !ruby/object:Gem::Version
320
+ version: '0.10'
321
+ - !ruby/object:Gem::Dependency
322
+ name: rake
323
+ requirement: !ruby/object:Gem::Requirement
324
+ requirements:
325
+ - - "~>"
326
+ - !ruby/object:Gem::Version
327
+ version: '10.4'
328
+ type: :development
329
+ prerelease: false
330
+ version_requirements: !ruby/object:Gem::Requirement
331
+ requirements:
332
+ - - "~>"
333
+ - !ruby/object:Gem::Version
334
+ version: '10.4'
335
+ - !ruby/object:Gem::Dependency
336
+ name: rspec
337
+ requirement: !ruby/object:Gem::Requirement
338
+ requirements:
339
+ - - "~>"
340
+ - !ruby/object:Gem::Version
341
+ version: '3.3'
342
+ type: :development
343
+ prerelease: false
344
+ version_requirements: !ruby/object:Gem::Requirement
345
+ requirements:
346
+ - - "~>"
347
+ - !ruby/object:Gem::Version
348
+ version: '3.3'
349
+ - !ruby/object:Gem::Dependency
350
+ name: sinatra
351
+ requirement: !ruby/object:Gem::Requirement
352
+ requirements:
353
+ - - "~>"
354
+ - !ruby/object:Gem::Version
355
+ version: '1.4'
356
+ type: :development
357
+ prerelease: false
358
+ version_requirements: !ruby/object:Gem::Requirement
359
+ requirements:
360
+ - - "~>"
361
+ - !ruby/object:Gem::Version
362
+ version: '1.4'
363
+ - !ruby/object:Gem::Dependency
364
+ name: rubocop
365
+ requirement: !ruby/object:Gem::Requirement
366
+ requirements:
367
+ - - "~>"
368
+ - !ruby/object:Gem::Version
369
+ version: '0.34'
370
+ type: :development
371
+ prerelease: false
372
+ version_requirements: !ruby/object:Gem::Requirement
373
+ requirements:
374
+ - - "~>"
375
+ - !ruby/object:Gem::Version
376
+ version: '0.34'
377
+ - !ruby/object:Gem::Dependency
378
+ name: yard
379
+ requirement: !ruby/object:Gem::Requirement
380
+ requirements:
381
+ - - "~>"
382
+ - !ruby/object:Gem::Version
383
+ version: '0.9'
384
+ type: :development
385
+ prerelease: false
386
+ version_requirements: !ruby/object:Gem::Requirement
387
+ requirements:
388
+ - - "~>"
389
+ - !ruby/object:Gem::Version
390
+ version: '0.9'
391
+ - !ruby/object:Gem::Dependency
392
+ name: bourbon
393
+ requirement: !ruby/object:Gem::Requirement
394
+ requirements:
395
+ - - "~>"
396
+ - !ruby/object:Gem::Version
397
+ version: '4.3'
398
+ type: :development
399
+ prerelease: false
400
+ version_requirements: !ruby/object:Gem::Requirement
401
+ requirements:
402
+ - - "~>"
403
+ - !ruby/object:Gem::Version
404
+ version: '4.3'
405
+ - !ruby/object:Gem::Dependency
406
+ name: neat
407
+ requirement: !ruby/object:Gem::Requirement
408
+ requirements:
409
+ - - "~>"
410
+ - !ruby/object:Gem::Version
411
+ version: '1.8'
412
+ type: :development
413
+ prerelease: false
414
+ version_requirements: !ruby/object:Gem::Requirement
415
+ requirements:
416
+ - - "~>"
417
+ - !ruby/object:Gem::Version
418
+ version: '1.8'
419
+ - !ruby/object:Gem::Dependency
420
+ name: faker
421
+ requirement: !ruby/object:Gem::Requirement
422
+ requirements:
423
+ - - "~>"
424
+ - !ruby/object:Gem::Version
425
+ version: '1.7'
426
+ type: :development
427
+ prerelease: false
428
+ version_requirements: !ruby/object:Gem::Requirement
429
+ requirements:
430
+ - - "~>"
431
+ - !ruby/object:Gem::Version
432
+ version: '1.7'
433
+ description: Versatile web crawling with Ruby
434
+ email: bauerdominic@gmail.com
435
+ executables:
436
+ - wayfarer
437
+ extensions: []
438
+ extra_rdoc_files: []
439
+ files:
440
+ - ".gitignore"
441
+ - ".rbenv-gemsets"
442
+ - ".rspec"
443
+ - ".rubocop.yml"
444
+ - ".ruby-version"
445
+ - ".travis.yml"
446
+ - ".yardopts"
447
+ - Changelog.md
448
+ - Gemfile
449
+ - LICENSE
450
+ - README.md
451
+ - Rakefile
452
+ - benchmark/frontiers.rb
453
+ - bin/wayfarer
454
+ - docs/.gitignore
455
+ - docs/_config.yml
456
+ - docs/_includes/base.html
457
+ - docs/_includes/head.html
458
+ - docs/_includes/navigation.html
459
+ - docs/_layouts/default.html
460
+ - docs/_sass/base.scss
461
+ - docs/_sass/variables.scss
462
+ - docs/_sass/vendor/bourbon/_bourbon-deprecate.scss
463
+ - docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss
464
+ - docs/_sass/vendor/bourbon/_bourbon.scss
465
+ - docs/_sass/vendor/bourbon/addons/_border-color.scss
466
+ - docs/_sass/vendor/bourbon/addons/_border-radius.scss
467
+ - docs/_sass/vendor/bourbon/addons/_border-style.scss
468
+ - docs/_sass/vendor/bourbon/addons/_border-width.scss
469
+ - docs/_sass/vendor/bourbon/addons/_buttons.scss
470
+ - docs/_sass/vendor/bourbon/addons/_clearfix.scss
471
+ - docs/_sass/vendor/bourbon/addons/_ellipsis.scss
472
+ - docs/_sass/vendor/bourbon/addons/_font-stacks.scss
473
+ - docs/_sass/vendor/bourbon/addons/_hide-text.scss
474
+ - docs/_sass/vendor/bourbon/addons/_margin.scss
475
+ - docs/_sass/vendor/bourbon/addons/_padding.scss
476
+ - docs/_sass/vendor/bourbon/addons/_position.scss
477
+ - docs/_sass/vendor/bourbon/addons/_prefixer.scss
478
+ - docs/_sass/vendor/bourbon/addons/_retina-image.scss
479
+ - docs/_sass/vendor/bourbon/addons/_size.scss
480
+ - docs/_sass/vendor/bourbon/addons/_text-inputs.scss
481
+ - docs/_sass/vendor/bourbon/addons/_timing-functions.scss
482
+ - docs/_sass/vendor/bourbon/addons/_triangle.scss
483
+ - docs/_sass/vendor/bourbon/addons/_word-wrap.scss
484
+ - docs/_sass/vendor/bourbon/css3/_animation.scss
485
+ - docs/_sass/vendor/bourbon/css3/_appearance.scss
486
+ - docs/_sass/vendor/bourbon/css3/_backface-visibility.scss
487
+ - docs/_sass/vendor/bourbon/css3/_background-image.scss
488
+ - docs/_sass/vendor/bourbon/css3/_background.scss
489
+ - docs/_sass/vendor/bourbon/css3/_border-image.scss
490
+ - docs/_sass/vendor/bourbon/css3/_calc.scss
491
+ - docs/_sass/vendor/bourbon/css3/_columns.scss
492
+ - docs/_sass/vendor/bourbon/css3/_filter.scss
493
+ - docs/_sass/vendor/bourbon/css3/_flex-box.scss
494
+ - docs/_sass/vendor/bourbon/css3/_font-face.scss
495
+ - docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss
496
+ - docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss
497
+ - docs/_sass/vendor/bourbon/css3/_hyphens.scss
498
+ - docs/_sass/vendor/bourbon/css3/_image-rendering.scss
499
+ - docs/_sass/vendor/bourbon/css3/_keyframes.scss
500
+ - docs/_sass/vendor/bourbon/css3/_linear-gradient.scss
501
+ - docs/_sass/vendor/bourbon/css3/_perspective.scss
502
+ - docs/_sass/vendor/bourbon/css3/_placeholder.scss
503
+ - docs/_sass/vendor/bourbon/css3/_radial-gradient.scss
504
+ - docs/_sass/vendor/bourbon/css3/_selection.scss
505
+ - docs/_sass/vendor/bourbon/css3/_text-decoration.scss
506
+ - docs/_sass/vendor/bourbon/css3/_transform.scss
507
+ - docs/_sass/vendor/bourbon/css3/_transition.scss
508
+ - docs/_sass/vendor/bourbon/css3/_user-select.scss
509
+ - docs/_sass/vendor/bourbon/functions/_assign-inputs.scss
510
+ - docs/_sass/vendor/bourbon/functions/_contains-falsy.scss
511
+ - docs/_sass/vendor/bourbon/functions/_contains.scss
512
+ - docs/_sass/vendor/bourbon/functions/_is-length.scss
513
+ - docs/_sass/vendor/bourbon/functions/_is-light.scss
514
+ - docs/_sass/vendor/bourbon/functions/_is-number.scss
515
+ - docs/_sass/vendor/bourbon/functions/_is-size.scss
516
+ - docs/_sass/vendor/bourbon/functions/_modular-scale.scss
517
+ - docs/_sass/vendor/bourbon/functions/_px-to-em.scss
518
+ - docs/_sass/vendor/bourbon/functions/_px-to-rem.scss
519
+ - docs/_sass/vendor/bourbon/functions/_shade.scss
520
+ - docs/_sass/vendor/bourbon/functions/_strip-units.scss
521
+ - docs/_sass/vendor/bourbon/functions/_tint.scss
522
+ - docs/_sass/vendor/bourbon/functions/_transition-property-name.scss
523
+ - docs/_sass/vendor/bourbon/functions/_unpack.scss
524
+ - docs/_sass/vendor/bourbon/helpers/_convert-units.scss
525
+ - docs/_sass/vendor/bourbon/helpers/_directional-values.scss
526
+ - docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss
527
+ - docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss
528
+ - docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss
529
+ - docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss
530
+ - docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss
531
+ - docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss
532
+ - docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss
533
+ - docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss
534
+ - docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss
535
+ - docs/_sass/vendor/bourbon/helpers/_render-gradients.scss
536
+ - docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss
537
+ - docs/_sass/vendor/bourbon/helpers/_str-to-num.scss
538
+ - docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss
539
+ - docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss
540
+ - docs/_sass/vendor/bourbon/settings/_prefixer.scss
541
+ - docs/_sass/vendor/bourbon/settings/_px-to-em.scss
542
+ - docs/_sass/vendor/neat/_neat-helpers.scss
543
+ - docs/_sass/vendor/neat/_neat.scss
544
+ - docs/_sass/vendor/neat/functions/_new-breakpoint.scss
545
+ - docs/_sass/vendor/neat/functions/_private.scss
546
+ - docs/_sass/vendor/neat/grid/_box-sizing.scss
547
+ - docs/_sass/vendor/neat/grid/_direction-context.scss
548
+ - docs/_sass/vendor/neat/grid/_display-context.scss
549
+ - docs/_sass/vendor/neat/grid/_fill-parent.scss
550
+ - docs/_sass/vendor/neat/grid/_media.scss
551
+ - docs/_sass/vendor/neat/grid/_omega.scss
552
+ - docs/_sass/vendor/neat/grid/_outer-container.scss
553
+ - docs/_sass/vendor/neat/grid/_pad.scss
554
+ - docs/_sass/vendor/neat/grid/_private.scss
555
+ - docs/_sass/vendor/neat/grid/_row.scss
556
+ - docs/_sass/vendor/neat/grid/_shift.scss
557
+ - docs/_sass/vendor/neat/grid/_span-columns.scss
558
+ - docs/_sass/vendor/neat/grid/_to-deprecate.scss
559
+ - docs/_sass/vendor/neat/grid/_visual-grid.scss
560
+ - docs/_sass/vendor/neat/mixins/_clearfix.scss
561
+ - docs/_sass/vendor/neat/settings/_disable-warnings.scss
562
+ - docs/_sass/vendor/neat/settings/_grid.scss
563
+ - docs/_sass/vendor/neat/settings/_visual-grid.scss
564
+ - docs/_sass/vendor/normalize-3.0.2.scss
565
+ - docs/_sass/vendor/pygments.scss
566
+ - docs/automating_browsers/capybara.md
567
+ - docs/css/screen.scss
568
+ - docs/guides/callbacks.md
569
+ - docs/guides/cli.md
570
+ - docs/guides/configuration.md
571
+ - docs/guides/error_handling.md
572
+ - docs/guides/frontiers.md
573
+ - docs/guides/halting.md
574
+ - docs/guides/job_queues.md
575
+ - docs/guides/locals.md
576
+ - docs/guides/logging.md
577
+ - docs/guides/page_objects.md
578
+ - docs/guides/peeking.md
579
+ - docs/guides/selenium_capybara.md
580
+ - docs/guides/tutorial.md
581
+ - docs/index.md
582
+ - docs/js/navigation.js
583
+ - docs/misc/contributing.md
584
+ - docs/misc/testing.md
585
+ - docs/recipes/authentication.md
586
+ - docs/recipes/csv.md
587
+ - docs/recipes/javascript.md
588
+ - docs/recipes/multiple_uris.md
589
+ - docs/recipes/screenshots.md
590
+ - docs/routing/custom_rules.md
591
+ - docs/routing/filetypes_rules.md
592
+ - docs/routing/host_rules.md
593
+ - docs/routing/path_rules.md
594
+ - docs/routing/protocol_rules.md
595
+ - docs/routing/query_rules.md
596
+ - docs/routing/routes.md
597
+ - docs/routing/uri_rules.md
598
+ - examples/collect_github_issues.rb
599
+ - examples/find_foobar_on_wikipedia.rb
600
+ - lib/wayfarer.rb
601
+ - lib/wayfarer/configuration.rb
602
+ - lib/wayfarer/crawl.rb
603
+ - lib/wayfarer/crawl_observer.rb
604
+ - lib/wayfarer/dispatcher.rb
605
+ - lib/wayfarer/finders.rb
606
+ - lib/wayfarer/frontiers/frontier.rb
607
+ - lib/wayfarer/frontiers/memory_bloomfilter.rb
608
+ - lib/wayfarer/frontiers/memory_frontier.rb
609
+ - lib/wayfarer/frontiers/memory_trie_frontier.rb
610
+ - lib/wayfarer/frontiers/normalize_uris.rb
611
+ - lib/wayfarer/frontiers/redis_bloomfilter.rb
612
+ - lib/wayfarer/frontiers/redis_frontier.rb
613
+ - lib/wayfarer/http_adapters/adapter_pool.rb
614
+ - lib/wayfarer/http_adapters/net_http_adapter.rb
615
+ - lib/wayfarer/http_adapters/selenium_adapter.rb
616
+ - lib/wayfarer/job.rb
617
+ - lib/wayfarer/locals.rb
618
+ - lib/wayfarer/page.rb
619
+ - lib/wayfarer/parsers/json_parser.rb
620
+ - lib/wayfarer/parsers/xml_parser.rb
621
+ - lib/wayfarer/processor.rb
622
+ - lib/wayfarer/routing/custom_rule.rb
623
+ - lib/wayfarer/routing/filetypes_rule.rb
624
+ - lib/wayfarer/routing/host_rule.rb
625
+ - lib/wayfarer/routing/path_rule.rb
626
+ - lib/wayfarer/routing/protocol_rule.rb
627
+ - lib/wayfarer/routing/query_rule.rb
628
+ - lib/wayfarer/routing/router.rb
629
+ - lib/wayfarer/routing/rule.rb
630
+ - lib/wayfarer/routing/uri_rule.rb
631
+ - spec/configuration_spec.rb
632
+ - spec/crawl_spec.rb
633
+ - spec/finders_spec.rb
634
+ - spec/frontiers/memory_bloomfilter_spec.rb
635
+ - spec/frontiers/memory_frontier_spec.rb
636
+ - spec/frontiers/memory_trie_frontier_spec.rb
637
+ - spec/frontiers/normalize_uris_spec.rb
638
+ - spec/frontiers/redis_bloomfilter_spec.rb
639
+ - spec/frontiers/redis_frontier_spec.rb
640
+ - spec/http_adapters/adapter_pool_spec.rb
641
+ - spec/http_adapters/net_http_adapter_spec.rb
642
+ - spec/http_adapters/selenium_adapter_spec.rb
643
+ - spec/integration/callbacks_spec.rb
644
+ - spec/integration/locals_spec.rb
645
+ - spec/integration/peeking_spec.rb
646
+ - spec/job_spec.rb
647
+ - spec/page_spec.rb
648
+ - spec/parsers/json_parser_spec.rb
649
+ - spec/parsers/xml_parser_spec.rb
650
+ - spec/processor_spec.rb
651
+ - spec/routing/custom_rule_spec.rb
652
+ - spec/routing/filetypes_rule_spec.rb
653
+ - spec/routing/host_rule_spec.rb
654
+ - spec/routing/path_rule_spec.rb
655
+ - spec/routing/protocol_rule_spec.rb
656
+ - spec/routing/query_rule_spec.rb
657
+ - spec/routing/router_spec.rb
658
+ - spec/routing/rule_spec.rb
659
+ - spec/routing/uri_rule_spec.rb
660
+ - spec/shared/frontier.rb
661
+ - spec/spec_helpers.rb
662
+ - spec/wayfarer_spec.rb
663
+ - support/static/finders.html
664
+ - support/static/graph/details/a.html
665
+ - support/static/graph/details/b.html
666
+ - support/static/graph/index.html
667
+ - support/static/json/dummy.json
668
+ - support/static/links/links.html
669
+ - support/static/xml/dummy.xml
670
+ - support/test_app.rb
671
+ - wayfarer-jruby.gemspec
672
+ - wayfarer.gemspec
673
+ homepage: http://github.com/bauerd/wayfarer
674
+ licenses:
675
+ - MIT
676
+ metadata: {}
677
+ post_install_message:
678
+ rdoc_options: []
679
+ require_paths:
680
+ - lib
681
+ required_ruby_version: !ruby/object:Gem::Requirement
682
+ requirements:
683
+ - - ">="
684
+ - !ruby/object:Gem::Version
685
+ version: '0'
686
+ required_rubygems_version: !ruby/object:Gem::Requirement
687
+ requirements:
688
+ - - ">="
689
+ - !ruby/object:Gem::Version
690
+ version: '0'
691
+ requirements: []
692
+ rubyforge_project:
693
+ rubygems_version: 2.5.1
694
+ signing_key:
695
+ specification_version: 4
696
+ summary: Versatile web crawling with Ruby
697
+ test_files: []