wayfarer 0.0.3 → 0.4.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (369) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/ci.yaml +32 -0
  3. data/.gitignore +3 -4
  4. data/.rubocop.yml +25 -9
  5. data/.ruby-version +1 -1
  6. data/Dockerfile +5 -0
  7. data/Gemfile +1 -7
  8. data/Gemfile.lock +221 -0
  9. data/RELEASING.md +17 -0
  10. data/Rakefile +38 -90
  11. data/bin/wayfarer +1 -111
  12. data/docker-compose.yml +32 -0
  13. data/docs/cookbook/batch_routing.md +22 -0
  14. data/docs/cookbook/consent_screen.md +36 -0
  15. data/docs/cookbook/executing_javascript.md +41 -0
  16. data/docs/cookbook/querying_html.md +42 -0
  17. data/docs/cookbook/screenshots.md +27 -0
  18. data/docs/cookbook/user_agent.md +7 -0
  19. data/docs/guides/browser_automation/capybara.md +69 -0
  20. data/docs/guides/browser_automation/custom_adapters.md +100 -0
  21. data/docs/guides/browser_automation/ferrum.md +39 -0
  22. data/docs/guides/browser_automation/selenium.md +63 -0
  23. data/docs/guides/callbacks.md +131 -31
  24. data/docs/guides/configuration.md +24 -169
  25. data/docs/guides/debugging.md +17 -0
  26. data/docs/guides/error_handling.md +30 -45
  27. data/docs/guides/jobs.md +101 -0
  28. data/docs/guides/navigation.md +73 -0
  29. data/docs/guides/networking.md +94 -0
  30. data/docs/guides/pages.md +52 -0
  31. data/docs/guides/performance.md +130 -0
  32. data/docs/guides/reliability.md +41 -0
  33. data/docs/guides/routing/steering.md +30 -0
  34. data/docs/guides/tasks.md +14 -0
  35. data/docs/index.md +40 -66
  36. data/docs/reference/api/base.md +48 -0
  37. data/docs/reference/api/route.md +182 -0
  38. data/docs/reference/cli.md +61 -0
  39. data/docs/reference/configuration_keys.md +42 -0
  40. data/docs/reference/environment_variables.md +83 -0
  41. data/lib/wayfarer/base.rb +50 -0
  42. data/lib/wayfarer/callbacks.rb +71 -0
  43. data/lib/wayfarer/cli/base.rb +27 -0
  44. data/lib/wayfarer/cli/generate.rb +17 -0
  45. data/lib/wayfarer/cli/job.rb +60 -0
  46. data/lib/wayfarer/cli/route.rb +29 -0
  47. data/lib/wayfarer/cli/route_printer.rb +116 -0
  48. data/lib/wayfarer/cli/runner.rb +34 -0
  49. data/lib/wayfarer/cli/templates/Gemfile.tt +5 -0
  50. data/lib/wayfarer/cli/templates/job.rb.tt +10 -0
  51. data/lib/wayfarer/config/capybara.rb +10 -0
  52. data/lib/wayfarer/config/ferrum.rb +11 -0
  53. data/lib/wayfarer/config/networking.rb +26 -0
  54. data/lib/wayfarer/config/redis.rb +14 -0
  55. data/lib/wayfarer/config/root.rb +11 -0
  56. data/lib/wayfarer/config/selenium.rb +21 -0
  57. data/lib/wayfarer/config/strconv.rb +45 -0
  58. data/lib/wayfarer/config/struct.rb +72 -0
  59. data/lib/wayfarer/gc.rb +15 -0
  60. data/lib/wayfarer/middleware/chain.rb +19 -0
  61. data/lib/wayfarer/middleware/dedup.rb +25 -0
  62. data/lib/wayfarer/middleware/fetch.rb +47 -0
  63. data/lib/wayfarer/middleware/normalize.rb +25 -0
  64. data/lib/wayfarer/middleware/router.rb +53 -0
  65. data/lib/wayfarer/middleware/stage.rb +23 -0
  66. data/lib/wayfarer/middleware/worker.rb +30 -0
  67. data/lib/wayfarer/networking/capybara.rb +28 -0
  68. data/lib/wayfarer/networking/context.rb +36 -0
  69. data/lib/wayfarer/networking/ferrum.rb +35 -0
  70. data/lib/wayfarer/networking/http.rb +34 -0
  71. data/lib/wayfarer/networking/pool.rb +40 -0
  72. data/lib/wayfarer/networking/result.rb +18 -0
  73. data/lib/wayfarer/networking/selenium.rb +43 -0
  74. data/lib/wayfarer/networking/strategy.rb +38 -0
  75. data/lib/wayfarer/page.rb +17 -74
  76. data/lib/wayfarer/parsing/json.rb +17 -0
  77. data/lib/wayfarer/parsing/xml.rb +17 -0
  78. data/lib/wayfarer/redis/.#barrier.rb +1 -0
  79. data/lib/wayfarer/redis/barrier.rb +36 -0
  80. data/lib/wayfarer/redis/connection.rb +13 -0
  81. data/lib/wayfarer/redis/counter.rb +29 -0
  82. data/lib/wayfarer/redis/pool.rb +20 -0
  83. data/lib/wayfarer/redis/version.rb +19 -0
  84. data/lib/wayfarer/routing/dsl.rb +57 -0
  85. data/lib/wayfarer/routing/matchers/custom.rb +25 -0
  86. data/lib/wayfarer/routing/matchers/host.rb +19 -0
  87. data/lib/wayfarer/routing/matchers/path.rb +49 -0
  88. data/lib/wayfarer/routing/matchers/query.rb +63 -0
  89. data/lib/wayfarer/routing/matchers/scheme.rb +17 -0
  90. data/lib/wayfarer/routing/matchers/suffix.rb +17 -0
  91. data/lib/wayfarer/routing/matchers/url.rb +17 -0
  92. data/lib/wayfarer/routing/path_finder.rb +46 -0
  93. data/lib/wayfarer/routing/result.rb +15 -0
  94. data/lib/wayfarer/routing/root_route.rb +7 -0
  95. data/lib/wayfarer/routing/route.rb +47 -0
  96. data/lib/wayfarer/routing/router.rb +10 -54
  97. data/lib/wayfarer/routing/target_route.rb +7 -0
  98. data/lib/wayfarer/serializer.rb +17 -0
  99. data/lib/wayfarer/stringify.rb +47 -0
  100. data/lib/wayfarer/task.rb +34 -0
  101. data/lib/wayfarer.rb +48 -57
  102. data/mkdocs.yml +47 -0
  103. data/requirements.txt +1 -0
  104. data/spec/base_spec.rb +233 -0
  105. data/spec/callbacks_spec.rb +102 -0
  106. data/spec/cli/generate_spec.rb +39 -0
  107. data/spec/cli/job_spec.rb +74 -0
  108. data/spec/cli/version_spec.rb +13 -0
  109. data/spec/config/capybara_spec.rb +18 -0
  110. data/spec/config/ferrum_spec.rb +24 -0
  111. data/spec/config/networking_spec.rb +73 -0
  112. data/spec/config/redis_spec.rb +32 -0
  113. data/spec/config/root_spec.rb +31 -0
  114. data/spec/config/selenium_spec.rb +56 -0
  115. data/spec/config/strconv_spec.rb +58 -0
  116. data/spec/config/struct_spec.rb +66 -0
  117. data/spec/factories/middleware.rb +15 -0
  118. data/spec/factories/page.rb +78 -0
  119. data/spec/factories/task.rb +12 -0
  120. data/spec/fixtures/dummy_job.rb +7 -0
  121. data/spec/gc_spec.rb +63 -0
  122. data/spec/middleware/chain_spec.rb +96 -0
  123. data/spec/middleware/dedup_spec.rb +76 -0
  124. data/spec/middleware/fetch_spec.rb +100 -0
  125. data/spec/middleware/normalize_spec.rb +28 -0
  126. data/spec/middleware/router_spec.rb +80 -0
  127. data/spec/middleware/stage_spec.rb +39 -0
  128. data/spec/middleware/worker_spec.rb +117 -0
  129. data/spec/networking/capybara_spec.rb +12 -0
  130. data/spec/networking/context_spec.rb +127 -0
  131. data/spec/networking/ferrum_spec.rb +12 -0
  132. data/spec/networking/http_spec.rb +12 -0
  133. data/spec/networking/pool_spec.rb +67 -0
  134. data/spec/networking/selenium_spec.rb +12 -0
  135. data/spec/networking/strategy.rb +170 -0
  136. data/spec/page_spec.rb +21 -12
  137. data/spec/{parsers/json_parser_spec.rb → parsing/json_spec.rb} +5 -4
  138. data/spec/{parsers/xml_parser_spec.rb → parsing/xml_spec.rb} +3 -2
  139. data/spec/redis/barrier_spec.rb +78 -0
  140. data/spec/redis/counter_spec.rb +32 -0
  141. data/spec/redis/pool_spec.rb +18 -0
  142. data/spec/redis/version_spec.rb +13 -0
  143. data/spec/routing/dsl_spec.rb +98 -0
  144. data/spec/routing/integration_spec.rb +110 -0
  145. data/spec/routing/matchers/custom_spec.rb +31 -0
  146. data/spec/routing/matchers/host_spec.rb +49 -0
  147. data/spec/routing/matchers/path_spec.rb +43 -0
  148. data/spec/routing/matchers/query_spec.rb +137 -0
  149. data/spec/routing/matchers/scheme_spec.rb +25 -0
  150. data/spec/routing/{filetypes_rule_spec.rb → matchers/suffix_spec.rb} +14 -13
  151. data/spec/routing/matchers/uri_spec.rb +27 -0
  152. data/spec/routing/path_finder_spec.rb +33 -0
  153. data/spec/routing/root_route_spec.rb +29 -0
  154. data/spec/routing/route_spec.rb +43 -0
  155. data/spec/routing/router_spec.rb +13 -56
  156. data/spec/spec_helpers.rb +73 -38
  157. data/spec/stringify_spec.rb +23 -0
  158. data/{support → spec/support}/static/finders.html +0 -0
  159. data/{support → spec/support}/static/graph/details/a.html +0 -0
  160. data/{support → spec/support}/static/graph/details/b.html +0 -0
  161. data/{support → spec/support}/static/graph/index.html +0 -0
  162. data/{support → spec/support}/static/json/dummy.json +0 -0
  163. data/{support → spec/support}/static/links/links.html +0 -0
  164. data/{support → spec/support}/static/xml/dummy.xml +0 -0
  165. data/{support → spec/support}/test_app.rb +9 -2
  166. data/spec/task_spec.rb +27 -0
  167. data/spec/wayfarer_spec.rb +2 -13
  168. data/wayfarer.gemspec +40 -42
  169. metadata +234 -361
  170. data/.travis.yml +0 -5
  171. data/Changelog.md +0 -10
  172. data/README.md +0 -21
  173. data/benchmark/frontiers.rb +0 -143
  174. data/docs/.gitignore +0 -2
  175. data/docs/_config.yml +0 -15
  176. data/docs/_includes/base.html +0 -7
  177. data/docs/_includes/head.html +0 -10
  178. data/docs/_includes/navigation.html +0 -187
  179. data/docs/_layouts/default.html +0 -42
  180. data/docs/_sass/base.scss +0 -439
  181. data/docs/_sass/variables.scss +0 -24
  182. data/docs/_sass/vendor/bourbon/_bourbon-deprecate.scss +0 -19
  183. data/docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss +0 -425
  184. data/docs/_sass/vendor/bourbon/_bourbon.scss +0 -90
  185. data/docs/_sass/vendor/bourbon/addons/_border-color.scss +0 -29
  186. data/docs/_sass/vendor/bourbon/addons/_border-radius.scss +0 -48
  187. data/docs/_sass/vendor/bourbon/addons/_border-style.scss +0 -28
  188. data/docs/_sass/vendor/bourbon/addons/_border-width.scss +0 -28
  189. data/docs/_sass/vendor/bourbon/addons/_buttons.scss +0 -69
  190. data/docs/_sass/vendor/bourbon/addons/_clearfix.scss +0 -25
  191. data/docs/_sass/vendor/bourbon/addons/_ellipsis.scss +0 -30
  192. data/docs/_sass/vendor/bourbon/addons/_font-stacks.scss +0 -31
  193. data/docs/_sass/vendor/bourbon/addons/_hide-text.scss +0 -27
  194. data/docs/_sass/vendor/bourbon/addons/_margin.scss +0 -29
  195. data/docs/_sass/vendor/bourbon/addons/_padding.scss +0 -29
  196. data/docs/_sass/vendor/bourbon/addons/_position.scss +0 -51
  197. data/docs/_sass/vendor/bourbon/addons/_prefixer.scss +0 -66
  198. data/docs/_sass/vendor/bourbon/addons/_retina-image.scss +0 -27
  199. data/docs/_sass/vendor/bourbon/addons/_size.scss +0 -56
  200. data/docs/_sass/vendor/bourbon/addons/_text-inputs.scss +0 -118
  201. data/docs/_sass/vendor/bourbon/addons/_timing-functions.scss +0 -34
  202. data/docs/_sass/vendor/bourbon/addons/_triangle.scss +0 -63
  203. data/docs/_sass/vendor/bourbon/addons/_word-wrap.scss +0 -29
  204. data/docs/_sass/vendor/bourbon/css3/_animation.scss +0 -61
  205. data/docs/_sass/vendor/bourbon/css3/_appearance.scss +0 -5
  206. data/docs/_sass/vendor/bourbon/css3/_backface-visibility.scss +0 -5
  207. data/docs/_sass/vendor/bourbon/css3/_background-image.scss +0 -44
  208. data/docs/_sass/vendor/bourbon/css3/_background.scss +0 -57
  209. data/docs/_sass/vendor/bourbon/css3/_border-image.scss +0 -61
  210. data/docs/_sass/vendor/bourbon/css3/_calc.scss +0 -6
  211. data/docs/_sass/vendor/bourbon/css3/_columns.scss +0 -67
  212. data/docs/_sass/vendor/bourbon/css3/_filter.scss +0 -6
  213. data/docs/_sass/vendor/bourbon/css3/_flex-box.scss +0 -327
  214. data/docs/_sass/vendor/bourbon/css3/_font-face.scss +0 -29
  215. data/docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss +0 -6
  216. data/docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss +0 -12
  217. data/docs/_sass/vendor/bourbon/css3/_hyphens.scss +0 -6
  218. data/docs/_sass/vendor/bourbon/css3/_image-rendering.scss +0 -15
  219. data/docs/_sass/vendor/bourbon/css3/_keyframes.scss +0 -38
  220. data/docs/_sass/vendor/bourbon/css3/_linear-gradient.scss +0 -40
  221. data/docs/_sass/vendor/bourbon/css3/_perspective.scss +0 -12
  222. data/docs/_sass/vendor/bourbon/css3/_placeholder.scss +0 -10
  223. data/docs/_sass/vendor/bourbon/css3/_radial-gradient.scss +0 -40
  224. data/docs/_sass/vendor/bourbon/css3/_selection.scss +0 -44
  225. data/docs/_sass/vendor/bourbon/css3/_text-decoration.scss +0 -27
  226. data/docs/_sass/vendor/bourbon/css3/_transform.scss +0 -21
  227. data/docs/_sass/vendor/bourbon/css3/_transition.scss +0 -81
  228. data/docs/_sass/vendor/bourbon/css3/_user-select.scss +0 -5
  229. data/docs/_sass/vendor/bourbon/functions/_assign-inputs.scss +0 -16
  230. data/docs/_sass/vendor/bourbon/functions/_contains-falsy.scss +0 -25
  231. data/docs/_sass/vendor/bourbon/functions/_contains.scss +0 -31
  232. data/docs/_sass/vendor/bourbon/functions/_is-length.scss +0 -16
  233. data/docs/_sass/vendor/bourbon/functions/_is-light.scss +0 -26
  234. data/docs/_sass/vendor/bourbon/functions/_is-number.scss +0 -16
  235. data/docs/_sass/vendor/bourbon/functions/_is-size.scss +0 -23
  236. data/docs/_sass/vendor/bourbon/functions/_modular-scale.scss +0 -74
  237. data/docs/_sass/vendor/bourbon/functions/_px-to-em.scss +0 -24
  238. data/docs/_sass/vendor/bourbon/functions/_px-to-rem.scss +0 -26
  239. data/docs/_sass/vendor/bourbon/functions/_shade.scss +0 -24
  240. data/docs/_sass/vendor/bourbon/functions/_strip-units.scss +0 -22
  241. data/docs/_sass/vendor/bourbon/functions/_tint.scss +0 -24
  242. data/docs/_sass/vendor/bourbon/functions/_transition-property-name.scss +0 -37
  243. data/docs/_sass/vendor/bourbon/functions/_unpack.scss +0 -32
  244. data/docs/_sass/vendor/bourbon/helpers/_convert-units.scss +0 -26
  245. data/docs/_sass/vendor/bourbon/helpers/_directional-values.scss +0 -108
  246. data/docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss +0 -53
  247. data/docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss +0 -24
  248. data/docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss +0 -35
  249. data/docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss +0 -51
  250. data/docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss +0 -77
  251. data/docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss +0 -41
  252. data/docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss +0 -74
  253. data/docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss +0 -55
  254. data/docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss +0 -28
  255. data/docs/_sass/vendor/bourbon/helpers/_render-gradients.scss +0 -31
  256. data/docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss +0 -15
  257. data/docs/_sass/vendor/bourbon/helpers/_str-to-num.scss +0 -55
  258. data/docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss +0 -7
  259. data/docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss +0 -8
  260. data/docs/_sass/vendor/bourbon/settings/_prefixer.scss +0 -9
  261. data/docs/_sass/vendor/bourbon/settings/_px-to-em.scss +0 -1
  262. data/docs/_sass/vendor/neat/_neat-helpers.scss +0 -11
  263. data/docs/_sass/vendor/neat/_neat.scss +0 -23
  264. data/docs/_sass/vendor/neat/functions/_new-breakpoint.scss +0 -49
  265. data/docs/_sass/vendor/neat/functions/_private.scss +0 -114
  266. data/docs/_sass/vendor/neat/grid/_box-sizing.scss +0 -15
  267. data/docs/_sass/vendor/neat/grid/_direction-context.scss +0 -33
  268. data/docs/_sass/vendor/neat/grid/_display-context.scss +0 -28
  269. data/docs/_sass/vendor/neat/grid/_fill-parent.scss +0 -22
  270. data/docs/_sass/vendor/neat/grid/_media.scss +0 -92
  271. data/docs/_sass/vendor/neat/grid/_omega.scss +0 -87
  272. data/docs/_sass/vendor/neat/grid/_outer-container.scss +0 -34
  273. data/docs/_sass/vendor/neat/grid/_pad.scss +0 -25
  274. data/docs/_sass/vendor/neat/grid/_private.scss +0 -35
  275. data/docs/_sass/vendor/neat/grid/_row.scss +0 -52
  276. data/docs/_sass/vendor/neat/grid/_shift.scss +0 -50
  277. data/docs/_sass/vendor/neat/grid/_span-columns.scss +0 -94
  278. data/docs/_sass/vendor/neat/grid/_to-deprecate.scss +0 -97
  279. data/docs/_sass/vendor/neat/grid/_visual-grid.scss +0 -42
  280. data/docs/_sass/vendor/neat/mixins/_clearfix.scss +0 -25
  281. data/docs/_sass/vendor/neat/settings/_disable-warnings.scss +0 -13
  282. data/docs/_sass/vendor/neat/settings/_grid.scss +0 -51
  283. data/docs/_sass/vendor/neat/settings/_visual-grid.scss +0 -27
  284. data/docs/_sass/vendor/normalize-3.0.2.scss +0 -427
  285. data/docs/_sass/vendor/pygments.scss +0 -356
  286. data/docs/automating_browsers/capybara.md +0 -70
  287. data/docs/css/screen.scss +0 -7
  288. data/docs/guides/cli.md +0 -52
  289. data/docs/guides/frontiers.md +0 -93
  290. data/docs/guides/halting.md +0 -23
  291. data/docs/guides/job_queues.md +0 -26
  292. data/docs/guides/locals.md +0 -36
  293. data/docs/guides/logging.md +0 -22
  294. data/docs/guides/page_objects.md +0 -67
  295. data/docs/guides/peeking.md +0 -46
  296. data/docs/guides/selenium_capybara.md +0 -100
  297. data/docs/guides/tutorial.md +0 -452
  298. data/docs/js/navigation.js +0 -11
  299. data/docs/misc/contributing.md +0 -20
  300. data/docs/misc/testing.md +0 -11
  301. data/docs/recipes/authentication.md +0 -23
  302. data/docs/recipes/csv.md +0 -29
  303. data/docs/recipes/javascript.md +0 -20
  304. data/docs/recipes/multiple_uris.md +0 -18
  305. data/docs/recipes/screenshots.md +0 -20
  306. data/docs/routing/custom_rules.md +0 -16
  307. data/docs/routing/filetypes_rules.md +0 -21
  308. data/docs/routing/host_rules.md +0 -24
  309. data/docs/routing/path_rules.md +0 -33
  310. data/docs/routing/protocol_rules.md +0 -17
  311. data/docs/routing/query_rules.md +0 -69
  312. data/docs/routing/routes.md +0 -96
  313. data/docs/routing/uri_rules.md +0 -18
  314. data/examples/collect_github_issues.rb +0 -65
  315. data/examples/find_foobar_on_wikipedia.rb +0 -23
  316. data/lib/wayfarer/configuration.rb +0 -86
  317. data/lib/wayfarer/crawl.rb +0 -79
  318. data/lib/wayfarer/crawl_observer.rb +0 -103
  319. data/lib/wayfarer/dispatcher.rb +0 -104
  320. data/lib/wayfarer/finders.rb +0 -61
  321. data/lib/wayfarer/frontiers/frontier.rb +0 -79
  322. data/lib/wayfarer/frontiers/memory_bloomfilter.rb +0 -32
  323. data/lib/wayfarer/frontiers/memory_frontier.rb +0 -76
  324. data/lib/wayfarer/frontiers/memory_trie_frontier.rb +0 -39
  325. data/lib/wayfarer/frontiers/normalize_uris.rb +0 -48
  326. data/lib/wayfarer/frontiers/redis_bloomfilter.rb +0 -34
  327. data/lib/wayfarer/frontiers/redis_frontier.rb +0 -83
  328. data/lib/wayfarer/http_adapters/adapter_pool.rb +0 -62
  329. data/lib/wayfarer/http_adapters/net_http_adapter.rb +0 -77
  330. data/lib/wayfarer/http_adapters/selenium_adapter.rb +0 -80
  331. data/lib/wayfarer/job.rb +0 -211
  332. data/lib/wayfarer/locals.rb +0 -40
  333. data/lib/wayfarer/parsers/json_parser.rb +0 -20
  334. data/lib/wayfarer/parsers/xml_parser.rb +0 -27
  335. data/lib/wayfarer/processor.rb +0 -103
  336. data/lib/wayfarer/routing/custom_rule.rb +0 -21
  337. data/lib/wayfarer/routing/filetypes_rule.rb +0 -20
  338. data/lib/wayfarer/routing/host_rule.rb +0 -19
  339. data/lib/wayfarer/routing/path_rule.rb +0 -54
  340. data/lib/wayfarer/routing/protocol_rule.rb +0 -21
  341. data/lib/wayfarer/routing/query_rule.rb +0 -59
  342. data/lib/wayfarer/routing/rule.rb +0 -114
  343. data/lib/wayfarer/routing/uri_rule.rb +0 -21
  344. data/spec/configuration_spec.rb +0 -26
  345. data/spec/crawl_spec.rb +0 -48
  346. data/spec/finders_spec.rb +0 -49
  347. data/spec/frontiers/memory_bloomfilter_spec.rb +0 -6
  348. data/spec/frontiers/memory_frontier_spec.rb +0 -6
  349. data/spec/frontiers/memory_trie_frontier_spec.rb +0 -6
  350. data/spec/frontiers/normalize_uris_spec.rb +0 -59
  351. data/spec/frontiers/redis_bloomfilter_spec.rb +0 -6
  352. data/spec/frontiers/redis_frontier_spec.rb +0 -6
  353. data/spec/http_adapters/adapter_pool_spec.rb +0 -33
  354. data/spec/http_adapters/net_http_adapter_spec.rb +0 -83
  355. data/spec/http_adapters/selenium_adapter_spec.rb +0 -53
  356. data/spec/integration/callbacks_spec.rb +0 -42
  357. data/spec/integration/locals_spec.rb +0 -106
  358. data/spec/integration/peeking_spec.rb +0 -61
  359. data/spec/job_spec.rb +0 -122
  360. data/spec/processor_spec.rb +0 -31
  361. data/spec/routing/custom_rule_spec.rb +0 -26
  362. data/spec/routing/host_rule_spec.rb +0 -48
  363. data/spec/routing/path_rule_spec.rb +0 -66
  364. data/spec/routing/protocol_rule_spec.rb +0 -26
  365. data/spec/routing/query_rule_spec.rb +0 -124
  366. data/spec/routing/rule_spec.rb +0 -251
  367. data/spec/routing/uri_rule_spec.rb +0 -24
  368. data/spec/shared/frontier.rb +0 -96
  369. data/wayfarer-jruby.gemspec +0 -49
@@ -1,54 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "mustermann"
4
-
5
- module Wayfarer
6
- module Routing
7
- # @private
8
- class PathRule < Rule
9
- attr_reader :matcher
10
-
11
- def initialize(arg, opts = {}, &proc)
12
- @matcher = if arg.is_a? String
13
- Mustermann.new(arg, type: Wayfarer.config.mustermann_type)
14
- else
15
- arg
16
- end
17
-
18
- super(opts, &proc)
19
- end
20
-
21
- def params(uri)
22
- return {} unless match!(uri)
23
-
24
- path = uri.path
25
-
26
- if @matcher.is_a? Mustermann
27
- @matcher.params(path)
28
- else
29
- captures = @matcher.match(full_path(uri)).captures
30
-
31
- captures.each.with_index.reduce({}) do |hash, (capture, i)|
32
- hash.merge(i.to_s => capture)
33
- end
34
- end
35
- end
36
-
37
- private
38
-
39
- # rubocop:disable Style/CaseEquality
40
- def match!(uri)
41
- if @matcher.is_a? Mustermann
42
- @matcher === uri.path
43
- else
44
- @matcher =~ full_path(uri)
45
- end
46
- end
47
- # rubocop:enable Style/CaseEquality
48
-
49
- def full_path(uri)
50
- "#{uri.path}?#{uri.query}##{uri.fragment}"
51
- end
52
- end
53
- end
54
- end
@@ -1,21 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "uri"
4
-
5
- module Wayfarer
6
- module Routing
7
- # @private
8
- class ProtocolRule < Rule
9
- def initialize(protocol, opts = {}, &proc)
10
- @protocol = protocol.to_s
11
- super(opts, &proc)
12
- end
13
-
14
- private
15
-
16
- def match!(uri)
17
- uri.scheme == @protocol
18
- end
19
- end
20
- end
21
- end
@@ -1,59 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "cgi"
4
-
5
- module Wayfarer
6
- module Routing
7
- # @private
8
- class QueryRule < Rule
9
- def initialize(field_constraints, opts = {}, &proc)
10
- @field_constraints = field_constraints
11
- super(opts, &proc)
12
- end
13
-
14
- private
15
-
16
- def match!(uri)
17
- CGI.parse(uri.query).none? do |field, vals| violates?(field, vals) end
18
- rescue NoMethodError
19
- # `CGI::parse` throws a `NoMethodError` if `uri.query` is an empty
20
- # string
21
- false
22
- end
23
-
24
- # rubocop:disable Lint/AssignmentInCondition
25
- def violates?(field, vals)
26
- return false unless constraint = @field_constraints[field.to_sym]
27
- violates_constraint?(constraint, vals)
28
- end
29
- # rubocop:enable Lint/AssignmentInCondition
30
-
31
- def violates_constraint?(constraint, vals)
32
- case constraint
33
- when String then violates_string?(constraint, vals)
34
- when Integer then violates_integer?(constraint, vals)
35
- when Regexp then violates_regexp?(constraint, vals)
36
- when Range then violates_range?(constraint, vals)
37
- end
38
- end
39
-
40
- def violates_string?(str, vals)
41
- vals.none? { |val| str == val }
42
- end
43
-
44
- def violates_integer?(int, vals)
45
- vals.none? do |val| int == Integer(val) end
46
- rescue ArgumentError
47
- true
48
- end
49
-
50
- def violates_regexp?(regexp, vals)
51
- vals.none? { |val| regexp.match(val) }
52
- end
53
-
54
- def violates_range?(range, vals)
55
- vals.none? { |val| range.include?(val.to_i) }
56
- end
57
- end
58
- end
59
- end
@@ -1,114 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "forwardable"
4
-
5
- module Wayfarer
6
- module Routing
7
- # Tree nodes
8
- # @private
9
- class Rule
10
- include Enumerable
11
-
12
- extend Forwardable
13
-
14
- attr_reader :child_rules
15
- attr_reader :target_action
16
-
17
- delegate [:each] => :child_rules
18
-
19
- def initialize(opts = {}, &proc)
20
- @child_rules = []
21
- @target_action = nil
22
-
23
- build_child_rule_chain_from_options(opts)
24
- instance_eval(&proc) if block_given?
25
- end
26
-
27
- def build_child_rule_chain_from_options(opts)
28
- @target_action = opts.delete(:to)
29
- opts.reduce(self) { |rule, (key, val)| rule.send(key, val) }
30
- end
31
-
32
- def matches?(uri)
33
- return false unless match!(uri)
34
- none? || any? { |child_rule| child_rule.matches?(uri) }
35
- end
36
-
37
- def invoke(uri)
38
- rule_chain = matching_rule_chain(uri)
39
-
40
- if rule_chain.any?
41
- params = params_from_rule_chain(rule_chain, uri)
42
- action = action_from_rule_chain(rule_chain)
43
-
44
- [true, params, action]
45
- else
46
- false
47
- end
48
- end
49
-
50
- # rubocop:disable Lint/AssignmentInCondition
51
- def matching_rule_chain(uri, chain = [])
52
- if match!(uri) && none?
53
- chain << self
54
- elsif matching_child = detect { |child_rule| child_rule.matches?(uri) }
55
- matching_child.matching_rule_chain(uri, chain << self)
56
- else
57
- []
58
- end
59
- end
60
- # rubocop:enable Lint/AssignmentInCondition
61
-
62
- def params(*)
63
- {}
64
- end
65
-
66
- def uri(*argv, &proc)
67
- append_child_rule(URIRule.new(*argv, &proc))
68
- end
69
-
70
- def host(*argv, &proc)
71
- append_child_rule(HostRule.new(*argv, &proc))
72
- end
73
-
74
- def path(*argv, &proc)
75
- append_child_rule(PathRule.new(*argv, &proc))
76
- end
77
-
78
- def query(*argv, &proc)
79
- append_child_rule(QueryRule.new(*argv, &proc))
80
- end
81
-
82
- def protocol(*argv, &proc)
83
- append_child_rule(ProtocolRule.new(*argv, &proc))
84
- end
85
-
86
- def if(*argv, &proc)
87
- append_child_rule(CustomRule.new(*argv, &proc))
88
- end
89
-
90
- def filetypes(*argv, &proc)
91
- append_child_rule(FiletypesRule.new(*argv, &proc))
92
- end
93
-
94
- private
95
-
96
- def append_child_rule(other)
97
- @child_rules << other
98
- other
99
- end
100
-
101
- def match!(*)
102
- any?
103
- end
104
-
105
- def params_from_rule_chain(rule_chain, uri)
106
- rule_chain.reduce({}) { |hash, rule| hash.merge(rule.params(uri)) }
107
- end
108
-
109
- def action_from_rule_chain(rule_chain)
110
- rule_chain.map(&:target_action).reverse.reject(&:nil?).first
111
- end
112
- end
113
- end
114
- end
@@ -1,21 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "uri"
4
-
5
- module Wayfarer
6
- module Routing
7
- # @private
8
- class URIRule < Rule
9
- def initialize(uri_str, opts = {}, &proc)
10
- @uri = URI(uri_str)
11
- super(opts, &proc)
12
- end
13
-
14
- private
15
-
16
- def match!(uri)
17
- uri == @uri
18
- end
19
- end
20
- end
21
- end
@@ -1,26 +0,0 @@
1
- # frozen_string_literal: true
2
- require "spec_helpers"
3
-
4
- describe Wayfarer::Configuration do
5
- let(:config) { Configuration.new }
6
-
7
- describe "::new" do
8
- it "overrides defaults" do
9
- config = Configuration.new(http_adapter: :selenium)
10
- expect(config.http_adapter).to be :selenium
11
- end
12
- end
13
-
14
- it "sets keys and values" do
15
- config.foo = :foo
16
- expect(config.foo).to be :foo
17
- end
18
-
19
- describe "#reset!" do
20
- it "resets to defaults" do
21
- config.max_http_redirects = 5
22
- config.reset!
23
- expect(config.max_http_redirects).to be 3
24
- end
25
- end
26
- end
data/spec/crawl_spec.rb DELETED
@@ -1,48 +0,0 @@
1
- # frozen_string_literal: true
2
- require "spec_helpers"
3
-
4
- describe Wayfarer::Crawl do
5
- let(:uris) { %w(http://example.com) }
6
- let(:job) { Class.new(Wayfarer::Job) }
7
- subject(:crawl) { Crawl.new(job, *uris) }
8
-
9
- describe "#frontier" do
10
- context "by default" do
11
- it "returns a MemoryFrontier" do
12
- expect(crawl.frontier).to be_a MemoryFrontier
13
- end
14
- end
15
-
16
- context "when Redis frontier is used", redis: true do
17
- before { job.config.frontier = :redis }
18
-
19
- it "returns a MemoryFrontier" do
20
- expect(crawl.frontier).to be_a RedisFrontier
21
- end
22
- end
23
-
24
- context "when memory bloomfilter is used", mri_only: true do
25
- before { job.config.frontier = :memory_bloom }
26
-
27
- it "returns a MemoryBloomfilter" do
28
- expect(crawl.frontier).to be_a MemoryBloomfilter
29
- end
30
- end
31
-
32
- context "when Redis bloomfilter is used", mri_only: true, redis: true do
33
- before { job.config.frontier = :redis_bloom }
34
-
35
- it "returns a RedisBloomfilter" do
36
- expect(crawl.frontier).to be_a RedisBloomfilter
37
- end
38
- end
39
-
40
- context "when memory trie is used", mri_only: true do
41
- before { job.config.frontier = :memory_trie }
42
-
43
- it "returns a RedisBloomfilter" do
44
- expect(crawl.frontier).to be_a MemoryTrieFrontier
45
- end
46
- end
47
- end
48
- end
data/spec/finders_spec.rb DELETED
@@ -1,49 +0,0 @@
1
- # frozen_string_literal: true
2
- require "spec_helpers"
3
-
4
- describe Wayfarer::Finders do
5
- let(:page) { fetch_page(test_app("/finders.html")) }
6
-
7
- describe "#links" do
8
- context "without paths" do
9
- it "returns all links" do
10
- expect(page.links.map(&:to_s)).to eq %w(
11
- http://localhost:9876/foo
12
- http://localhost:9876/bar
13
- http://localhost:9876/baz
14
- http://google.com
15
- http://yahoo.com
16
- http://aol.com
17
- )
18
- end
19
- end
20
-
21
- context "with paths" do
22
- it "returns targeted links" do
23
- expect(page.links("ul li:nth-child(3) a").map(&:to_s)).to eq %w(
24
- http://localhost:9876/baz
25
- )
26
- end
27
- end
28
- end
29
-
30
- describe "#stylesheets" do
31
- context "without paths" do
32
- it "returns all stylesheets" do
33
- expect(page.stylesheets.map(&:to_s)).to eq %w(
34
- http://localhost:9876/a.css
35
- http://localhost:9876/b.css
36
- http://google.com/c.css
37
- )
38
- end
39
- end
40
-
41
- context "with paths" do
42
- it "returns targeted stylesheets" do
43
- expect(page.stylesheets("#stylesheet-c").map(&:to_s)).to eq %w(
44
- http://google.com/c.css
45
- )
46
- end
47
- end
48
- end
49
- end
@@ -1,6 +0,0 @@
1
- # frozen_string_literal: true
2
- require "spec_helpers"
3
-
4
- describe Wayfarer::Frontiers::MemoryBloomfilter, mri_only: true do
5
- it_behaves_like "Frontier", MemoryBloomfilter
6
- end
@@ -1,6 +0,0 @@
1
- # frozen_string_literal: true
2
- require "spec_helpers"
3
-
4
- describe Wayfarer::Frontiers::MemoryFrontier do
5
- it_behaves_like "Frontier", MemoryFrontier
6
- end
@@ -1,6 +0,0 @@
1
- # frozen_string_literal: true
2
- require "spec_helpers"
3
-
4
- describe Wayfarer::Frontiers::MemoryTrieFrontier, mri_only: true do
5
- it_behaves_like "Frontier", MemoryTrieFrontier
6
- end
@@ -1,59 +0,0 @@
1
- # frozen_string_literal: true
2
- require "spec_helpers"
3
-
4
- describe Wayfarer::Frontiers::NormalizeURIs, mri_only: true do
5
- let(:config) { Wayfarer.config }
6
-
7
- let(:uris) do
8
- %w(
9
- http://example.com/foo?bar=1&zet=0
10
- http://example.com//foo?zet=0&bar=1
11
- http://example.com/foo?bar=1&zet=0#bar
12
- )
13
- end
14
-
15
- subject(:frontier) do
16
- MemoryFrontier.new(config).extend(NormalizeURIs)
17
- end
18
-
19
- describe "#stage" do
20
- it "normalizes URIs" do
21
- expect {
22
- frontier.stage(*uris)
23
- }.to change { frontier.staged_uris.count }.by(1)
24
- end
25
- end
26
-
27
- describe "#staged?" do
28
- it "normalizes URIs" do
29
- frontier.stage(*uris)
30
- uris.each { |u| expect(frontier.staged?(u)).to be true }
31
- end
32
- end
33
-
34
- describe "#cache, #cached?" do
35
- they "normalize URIs" do
36
- frontier.cache(*uris)
37
- uris.each { |u| expect(frontier.cached?(u)).to be true }
38
- end
39
- end
40
-
41
- context "with normalization options" do
42
- let(:config) do
43
- Configuration.new(normalize_uri_options: { remove_hash: false })
44
- end
45
-
46
- let(:uris) do
47
- %w(
48
- http://example.com/foo
49
- http://example.com/foo#bar
50
- )
51
- end
52
-
53
- it "adheres to options" do
54
- expect {
55
- frontier.stage(*uris)
56
- }.to change { frontier.staged_uris.count }.by(2)
57
- end
58
- end
59
- end
@@ -1,6 +0,0 @@
1
- # frozen_string_literal: true
2
- require "spec_helpers"
3
-
4
- describe Wayfarer::Frontiers::RedisBloomfilter, redis: true, mri_only: true do
5
- it_behaves_like "Frontier", RedisBloomfilter
6
- end
@@ -1,6 +0,0 @@
1
- # frozen_string_literal: true
2
- require "spec_helpers"
3
-
4
- describe Wayfarer::Frontiers::RedisFrontier, redis: true do
5
- it_behaves_like "Frontier", RedisFrontier
6
- end
@@ -1,33 +0,0 @@
1
- # frozen_string_literal: true
2
- require "spec_helpers"
3
-
4
- describe Wayfarer::HTTPAdapters::AdapterPool do
5
- subject(:adapter_pool) { AdapterPool.new(Wayfarer::Job) }
6
-
7
- describe "#with" do
8
- after { adapter_pool.free }
9
-
10
- context "when using Net::HTTP" do
11
- it "yields a NetHTTPAdapter" do
12
- adapter_pool.with do |adapter|
13
- expect(adapter).to be_a NetHTTPAdapter
14
- end
15
- end
16
- end
17
-
18
- context "when using Selenium", selenium: true do
19
- subject(:adapter_pool) do
20
- job = Class.new(Wayfarer::Job)
21
- job.config.http_adapter = :selenium
22
-
23
- AdapterPool.new(job)
24
- end
25
-
26
- it "yields a SeleniumAdapter" do
27
- adapter_pool.with do |adapter|
28
- expect(adapter).to be_a SeleniumAdapter
29
- end
30
- end
31
- end
32
- end
33
- end
@@ -1,83 +0,0 @@
1
- # frozen_string_literal: true
2
- require "spec_helpers"
3
-
4
- describe Wayfarer::HTTPAdapters::NetHTTPAdapter do
5
- subject(:adapter) { Wayfarer::HTTPAdapters::NetHTTPAdapter.instance }
6
-
7
- describe "#fetch" do
8
- it "returns a Page" do
9
- uri = test_app("/hello_world")
10
- page = adapter.fetch(uri)
11
- expect(page).to be_a Page
12
- end
13
-
14
- it "sets the correct URI" do
15
- uri = test_app("/status_code/404")
16
- page = adapter.fetch(uri)
17
- expect(page.uri).to eq test_app("/status_code/404")
18
- end
19
-
20
- it "retrieves the correct HTTP status code" do
21
- uri = test_app("/status_code/404")
22
- page = adapter.fetch(uri)
23
- expect(page.status_code).to be 404
24
- end
25
-
26
- it "retrieves the correct response body" do
27
- uri = test_app("/hello_world")
28
- page = adapter.fetch(uri)
29
- expect(page.body).to eq "Hello world!"
30
- end
31
-
32
- it "retrieves the correct response headers" do
33
- uri = test_app("/hello_world")
34
- page = adapter.fetch(uri)
35
- expect(page.headers["hello"]).to eq ["world"]
36
- end
37
-
38
- context "with malformed URI" do
39
- it "raises a MalformedURI" do
40
- expect do
41
- uri = URI("hptt://bro.ken")
42
- page = adapter.fetch(uri)
43
- end.to raise_error(
44
- Wayfarer::HTTPAdapters::NetHTTPAdapter::MalformedURI
45
- )
46
- end
47
- end
48
-
49
- context "when response is a redirect" do
50
- it "follows the redirect" do
51
- uri = test_app("/redirect?times=3")
52
- page = adapter.fetch(uri)
53
-
54
- expect(page.uri.to_s).to eq "http://localhost:9876/redirect?times=0"
55
- end
56
-
57
- context "when maximum number of redirects reached" do
58
- before { Wayfarer.config.max_http_redirects = 5 }
59
- after { Wayfarer.config.reset! }
60
-
61
- it "raises a MaximumRedirectCountReached" do
62
- expect do
63
- uri = test_app("/redirect?times=6")
64
- page = adapter.fetch(uri)
65
- end.to raise_error(
66
- Wayfarer::HTTPAdapters::NetHTTPAdapter::MaximumRedirectCountReached
67
- )
68
- end
69
- end
70
-
71
- context "when redirection URI is malformed" do
72
- it "raises a MalformedRedirectURI" do
73
- expect do
74
- uri = test_app("/malformed_redirect")
75
- page = adapter.fetch(uri)
76
- end.to raise_error(
77
- Wayfarer::HTTPAdapters::NetHTTPAdapter::MalformedRedirectURI
78
- )
79
- end
80
- end
81
- end
82
- end
83
- end
@@ -1,53 +0,0 @@
1
- # frozen_string_literal: true
2
- require "spec_helpers"
3
-
4
- describe Wayfarer::HTTPAdapters::SeleniumAdapter, selenium: true do
5
- subject(:adapter) { Wayfarer::HTTPAdapters::SeleniumAdapter.new }
6
- after { adapter.free }
7
-
8
- describe "#fetch" do
9
- it "returns a Page" do
10
- uri = URI("http://0.0.0.0:9876/hello_world")
11
- page = adapter.fetch(uri)
12
- expect(page).to be_a Page
13
- end
14
-
15
- it "sets the correct URI" do
16
- uri = URI("http://0.0.0.0:9876/status_code/404")
17
- page = adapter.fetch(uri)
18
- expect(page.uri.to_s).to eq "http://0.0.0.0:9876/status_code/404"
19
- end
20
-
21
- it "retrieves the correct HTTP status code" do
22
- uri = URI("http://0.0.0.0:9876/status_code/404")
23
- page = adapter.fetch(uri)
24
- expect(page.status_code).to be 404
25
- end
26
-
27
- it "retrieves the correct response body" do
28
- uri = URI("http://0.0.0.0:9876/hello_world")
29
- page = adapter.fetch(uri)
30
- expect(page.body).to match /Hello world!/
31
- end
32
-
33
- it "retrieves the correct response headers" do
34
- uri = test_app("/hello_world")
35
- page = adapter.fetch(uri)
36
- expect(page.headers["hello"]).to eq ["world"]
37
- end
38
-
39
- context "when response is a redirect" do
40
- it "follows the redirect" do
41
- uri = test_app("/redirect?times=3")
42
- page = adapter.fetch(uri)
43
- expect(page.uri.to_s).to eq test_app("/redirect?times=0").to_s
44
- end
45
-
46
- context "when encountering a redirect loop" do
47
- it "returns `nil` as HTTP status code" do
48
- pending; fail
49
- end
50
- end
51
- end
52
- end
53
- end