wayfarer 0.0.3 → 0.4.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (369) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/ci.yaml +32 -0
  3. data/.gitignore +3 -4
  4. data/.rubocop.yml +25 -9
  5. data/.ruby-version +1 -1
  6. data/Dockerfile +5 -0
  7. data/Gemfile +1 -7
  8. data/Gemfile.lock +221 -0
  9. data/RELEASING.md +17 -0
  10. data/Rakefile +38 -90
  11. data/bin/wayfarer +1 -111
  12. data/docker-compose.yml +32 -0
  13. data/docs/cookbook/batch_routing.md +22 -0
  14. data/docs/cookbook/consent_screen.md +36 -0
  15. data/docs/cookbook/executing_javascript.md +41 -0
  16. data/docs/cookbook/querying_html.md +42 -0
  17. data/docs/cookbook/screenshots.md +27 -0
  18. data/docs/cookbook/user_agent.md +7 -0
  19. data/docs/guides/browser_automation/capybara.md +69 -0
  20. data/docs/guides/browser_automation/custom_adapters.md +100 -0
  21. data/docs/guides/browser_automation/ferrum.md +39 -0
  22. data/docs/guides/browser_automation/selenium.md +63 -0
  23. data/docs/guides/callbacks.md +131 -31
  24. data/docs/guides/configuration.md +24 -169
  25. data/docs/guides/debugging.md +17 -0
  26. data/docs/guides/error_handling.md +30 -45
  27. data/docs/guides/jobs.md +101 -0
  28. data/docs/guides/navigation.md +73 -0
  29. data/docs/guides/networking.md +94 -0
  30. data/docs/guides/pages.md +52 -0
  31. data/docs/guides/performance.md +130 -0
  32. data/docs/guides/reliability.md +41 -0
  33. data/docs/guides/routing/steering.md +30 -0
  34. data/docs/guides/tasks.md +14 -0
  35. data/docs/index.md +40 -66
  36. data/docs/reference/api/base.md +48 -0
  37. data/docs/reference/api/route.md +182 -0
  38. data/docs/reference/cli.md +61 -0
  39. data/docs/reference/configuration_keys.md +42 -0
  40. data/docs/reference/environment_variables.md +83 -0
  41. data/lib/wayfarer/base.rb +50 -0
  42. data/lib/wayfarer/callbacks.rb +71 -0
  43. data/lib/wayfarer/cli/base.rb +27 -0
  44. data/lib/wayfarer/cli/generate.rb +17 -0
  45. data/lib/wayfarer/cli/job.rb +60 -0
  46. data/lib/wayfarer/cli/route.rb +29 -0
  47. data/lib/wayfarer/cli/route_printer.rb +116 -0
  48. data/lib/wayfarer/cli/runner.rb +34 -0
  49. data/lib/wayfarer/cli/templates/Gemfile.tt +5 -0
  50. data/lib/wayfarer/cli/templates/job.rb.tt +10 -0
  51. data/lib/wayfarer/config/capybara.rb +10 -0
  52. data/lib/wayfarer/config/ferrum.rb +11 -0
  53. data/lib/wayfarer/config/networking.rb +26 -0
  54. data/lib/wayfarer/config/redis.rb +14 -0
  55. data/lib/wayfarer/config/root.rb +11 -0
  56. data/lib/wayfarer/config/selenium.rb +21 -0
  57. data/lib/wayfarer/config/strconv.rb +45 -0
  58. data/lib/wayfarer/config/struct.rb +72 -0
  59. data/lib/wayfarer/gc.rb +15 -0
  60. data/lib/wayfarer/middleware/chain.rb +19 -0
  61. data/lib/wayfarer/middleware/dedup.rb +25 -0
  62. data/lib/wayfarer/middleware/fetch.rb +47 -0
  63. data/lib/wayfarer/middleware/normalize.rb +25 -0
  64. data/lib/wayfarer/middleware/router.rb +53 -0
  65. data/lib/wayfarer/middleware/stage.rb +23 -0
  66. data/lib/wayfarer/middleware/worker.rb +30 -0
  67. data/lib/wayfarer/networking/capybara.rb +28 -0
  68. data/lib/wayfarer/networking/context.rb +36 -0
  69. data/lib/wayfarer/networking/ferrum.rb +35 -0
  70. data/lib/wayfarer/networking/http.rb +34 -0
  71. data/lib/wayfarer/networking/pool.rb +40 -0
  72. data/lib/wayfarer/networking/result.rb +18 -0
  73. data/lib/wayfarer/networking/selenium.rb +43 -0
  74. data/lib/wayfarer/networking/strategy.rb +38 -0
  75. data/lib/wayfarer/page.rb +17 -74
  76. data/lib/wayfarer/parsing/json.rb +17 -0
  77. data/lib/wayfarer/parsing/xml.rb +17 -0
  78. data/lib/wayfarer/redis/.#barrier.rb +1 -0
  79. data/lib/wayfarer/redis/barrier.rb +36 -0
  80. data/lib/wayfarer/redis/connection.rb +13 -0
  81. data/lib/wayfarer/redis/counter.rb +29 -0
  82. data/lib/wayfarer/redis/pool.rb +20 -0
  83. data/lib/wayfarer/redis/version.rb +19 -0
  84. data/lib/wayfarer/routing/dsl.rb +57 -0
  85. data/lib/wayfarer/routing/matchers/custom.rb +25 -0
  86. data/lib/wayfarer/routing/matchers/host.rb +19 -0
  87. data/lib/wayfarer/routing/matchers/path.rb +49 -0
  88. data/lib/wayfarer/routing/matchers/query.rb +63 -0
  89. data/lib/wayfarer/routing/matchers/scheme.rb +17 -0
  90. data/lib/wayfarer/routing/matchers/suffix.rb +17 -0
  91. data/lib/wayfarer/routing/matchers/url.rb +17 -0
  92. data/lib/wayfarer/routing/path_finder.rb +46 -0
  93. data/lib/wayfarer/routing/result.rb +15 -0
  94. data/lib/wayfarer/routing/root_route.rb +7 -0
  95. data/lib/wayfarer/routing/route.rb +47 -0
  96. data/lib/wayfarer/routing/router.rb +10 -54
  97. data/lib/wayfarer/routing/target_route.rb +7 -0
  98. data/lib/wayfarer/serializer.rb +17 -0
  99. data/lib/wayfarer/stringify.rb +47 -0
  100. data/lib/wayfarer/task.rb +34 -0
  101. data/lib/wayfarer.rb +48 -57
  102. data/mkdocs.yml +47 -0
  103. data/requirements.txt +1 -0
  104. data/spec/base_spec.rb +233 -0
  105. data/spec/callbacks_spec.rb +102 -0
  106. data/spec/cli/generate_spec.rb +39 -0
  107. data/spec/cli/job_spec.rb +74 -0
  108. data/spec/cli/version_spec.rb +13 -0
  109. data/spec/config/capybara_spec.rb +18 -0
  110. data/spec/config/ferrum_spec.rb +24 -0
  111. data/spec/config/networking_spec.rb +73 -0
  112. data/spec/config/redis_spec.rb +32 -0
  113. data/spec/config/root_spec.rb +31 -0
  114. data/spec/config/selenium_spec.rb +56 -0
  115. data/spec/config/strconv_spec.rb +58 -0
  116. data/spec/config/struct_spec.rb +66 -0
  117. data/spec/factories/middleware.rb +15 -0
  118. data/spec/factories/page.rb +78 -0
  119. data/spec/factories/task.rb +12 -0
  120. data/spec/fixtures/dummy_job.rb +7 -0
  121. data/spec/gc_spec.rb +63 -0
  122. data/spec/middleware/chain_spec.rb +96 -0
  123. data/spec/middleware/dedup_spec.rb +76 -0
  124. data/spec/middleware/fetch_spec.rb +100 -0
  125. data/spec/middleware/normalize_spec.rb +28 -0
  126. data/spec/middleware/router_spec.rb +80 -0
  127. data/spec/middleware/stage_spec.rb +39 -0
  128. data/spec/middleware/worker_spec.rb +117 -0
  129. data/spec/networking/capybara_spec.rb +12 -0
  130. data/spec/networking/context_spec.rb +127 -0
  131. data/spec/networking/ferrum_spec.rb +12 -0
  132. data/spec/networking/http_spec.rb +12 -0
  133. data/spec/networking/pool_spec.rb +67 -0
  134. data/spec/networking/selenium_spec.rb +12 -0
  135. data/spec/networking/strategy.rb +170 -0
  136. data/spec/page_spec.rb +21 -12
  137. data/spec/{parsers/json_parser_spec.rb → parsing/json_spec.rb} +5 -4
  138. data/spec/{parsers/xml_parser_spec.rb → parsing/xml_spec.rb} +3 -2
  139. data/spec/redis/barrier_spec.rb +78 -0
  140. data/spec/redis/counter_spec.rb +32 -0
  141. data/spec/redis/pool_spec.rb +18 -0
  142. data/spec/redis/version_spec.rb +13 -0
  143. data/spec/routing/dsl_spec.rb +98 -0
  144. data/spec/routing/integration_spec.rb +110 -0
  145. data/spec/routing/matchers/custom_spec.rb +31 -0
  146. data/spec/routing/matchers/host_spec.rb +49 -0
  147. data/spec/routing/matchers/path_spec.rb +43 -0
  148. data/spec/routing/matchers/query_spec.rb +137 -0
  149. data/spec/routing/matchers/scheme_spec.rb +25 -0
  150. data/spec/routing/{filetypes_rule_spec.rb → matchers/suffix_spec.rb} +14 -13
  151. data/spec/routing/matchers/uri_spec.rb +27 -0
  152. data/spec/routing/path_finder_spec.rb +33 -0
  153. data/spec/routing/root_route_spec.rb +29 -0
  154. data/spec/routing/route_spec.rb +43 -0
  155. data/spec/routing/router_spec.rb +13 -56
  156. data/spec/spec_helpers.rb +73 -38
  157. data/spec/stringify_spec.rb +23 -0
  158. data/{support → spec/support}/static/finders.html +0 -0
  159. data/{support → spec/support}/static/graph/details/a.html +0 -0
  160. data/{support → spec/support}/static/graph/details/b.html +0 -0
  161. data/{support → spec/support}/static/graph/index.html +0 -0
  162. data/{support → spec/support}/static/json/dummy.json +0 -0
  163. data/{support → spec/support}/static/links/links.html +0 -0
  164. data/{support → spec/support}/static/xml/dummy.xml +0 -0
  165. data/{support → spec/support}/test_app.rb +9 -2
  166. data/spec/task_spec.rb +27 -0
  167. data/spec/wayfarer_spec.rb +2 -13
  168. data/wayfarer.gemspec +40 -42
  169. metadata +234 -361
  170. data/.travis.yml +0 -5
  171. data/Changelog.md +0 -10
  172. data/README.md +0 -21
  173. data/benchmark/frontiers.rb +0 -143
  174. data/docs/.gitignore +0 -2
  175. data/docs/_config.yml +0 -15
  176. data/docs/_includes/base.html +0 -7
  177. data/docs/_includes/head.html +0 -10
  178. data/docs/_includes/navigation.html +0 -187
  179. data/docs/_layouts/default.html +0 -42
  180. data/docs/_sass/base.scss +0 -439
  181. data/docs/_sass/variables.scss +0 -24
  182. data/docs/_sass/vendor/bourbon/_bourbon-deprecate.scss +0 -19
  183. data/docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss +0 -425
  184. data/docs/_sass/vendor/bourbon/_bourbon.scss +0 -90
  185. data/docs/_sass/vendor/bourbon/addons/_border-color.scss +0 -29
  186. data/docs/_sass/vendor/bourbon/addons/_border-radius.scss +0 -48
  187. data/docs/_sass/vendor/bourbon/addons/_border-style.scss +0 -28
  188. data/docs/_sass/vendor/bourbon/addons/_border-width.scss +0 -28
  189. data/docs/_sass/vendor/bourbon/addons/_buttons.scss +0 -69
  190. data/docs/_sass/vendor/bourbon/addons/_clearfix.scss +0 -25
  191. data/docs/_sass/vendor/bourbon/addons/_ellipsis.scss +0 -30
  192. data/docs/_sass/vendor/bourbon/addons/_font-stacks.scss +0 -31
  193. data/docs/_sass/vendor/bourbon/addons/_hide-text.scss +0 -27
  194. data/docs/_sass/vendor/bourbon/addons/_margin.scss +0 -29
  195. data/docs/_sass/vendor/bourbon/addons/_padding.scss +0 -29
  196. data/docs/_sass/vendor/bourbon/addons/_position.scss +0 -51
  197. data/docs/_sass/vendor/bourbon/addons/_prefixer.scss +0 -66
  198. data/docs/_sass/vendor/bourbon/addons/_retina-image.scss +0 -27
  199. data/docs/_sass/vendor/bourbon/addons/_size.scss +0 -56
  200. data/docs/_sass/vendor/bourbon/addons/_text-inputs.scss +0 -118
  201. data/docs/_sass/vendor/bourbon/addons/_timing-functions.scss +0 -34
  202. data/docs/_sass/vendor/bourbon/addons/_triangle.scss +0 -63
  203. data/docs/_sass/vendor/bourbon/addons/_word-wrap.scss +0 -29
  204. data/docs/_sass/vendor/bourbon/css3/_animation.scss +0 -61
  205. data/docs/_sass/vendor/bourbon/css3/_appearance.scss +0 -5
  206. data/docs/_sass/vendor/bourbon/css3/_backface-visibility.scss +0 -5
  207. data/docs/_sass/vendor/bourbon/css3/_background-image.scss +0 -44
  208. data/docs/_sass/vendor/bourbon/css3/_background.scss +0 -57
  209. data/docs/_sass/vendor/bourbon/css3/_border-image.scss +0 -61
  210. data/docs/_sass/vendor/bourbon/css3/_calc.scss +0 -6
  211. data/docs/_sass/vendor/bourbon/css3/_columns.scss +0 -67
  212. data/docs/_sass/vendor/bourbon/css3/_filter.scss +0 -6
  213. data/docs/_sass/vendor/bourbon/css3/_flex-box.scss +0 -327
  214. data/docs/_sass/vendor/bourbon/css3/_font-face.scss +0 -29
  215. data/docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss +0 -6
  216. data/docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss +0 -12
  217. data/docs/_sass/vendor/bourbon/css3/_hyphens.scss +0 -6
  218. data/docs/_sass/vendor/bourbon/css3/_image-rendering.scss +0 -15
  219. data/docs/_sass/vendor/bourbon/css3/_keyframes.scss +0 -38
  220. data/docs/_sass/vendor/bourbon/css3/_linear-gradient.scss +0 -40
  221. data/docs/_sass/vendor/bourbon/css3/_perspective.scss +0 -12
  222. data/docs/_sass/vendor/bourbon/css3/_placeholder.scss +0 -10
  223. data/docs/_sass/vendor/bourbon/css3/_radial-gradient.scss +0 -40
  224. data/docs/_sass/vendor/bourbon/css3/_selection.scss +0 -44
  225. data/docs/_sass/vendor/bourbon/css3/_text-decoration.scss +0 -27
  226. data/docs/_sass/vendor/bourbon/css3/_transform.scss +0 -21
  227. data/docs/_sass/vendor/bourbon/css3/_transition.scss +0 -81
  228. data/docs/_sass/vendor/bourbon/css3/_user-select.scss +0 -5
  229. data/docs/_sass/vendor/bourbon/functions/_assign-inputs.scss +0 -16
  230. data/docs/_sass/vendor/bourbon/functions/_contains-falsy.scss +0 -25
  231. data/docs/_sass/vendor/bourbon/functions/_contains.scss +0 -31
  232. data/docs/_sass/vendor/bourbon/functions/_is-length.scss +0 -16
  233. data/docs/_sass/vendor/bourbon/functions/_is-light.scss +0 -26
  234. data/docs/_sass/vendor/bourbon/functions/_is-number.scss +0 -16
  235. data/docs/_sass/vendor/bourbon/functions/_is-size.scss +0 -23
  236. data/docs/_sass/vendor/bourbon/functions/_modular-scale.scss +0 -74
  237. data/docs/_sass/vendor/bourbon/functions/_px-to-em.scss +0 -24
  238. data/docs/_sass/vendor/bourbon/functions/_px-to-rem.scss +0 -26
  239. data/docs/_sass/vendor/bourbon/functions/_shade.scss +0 -24
  240. data/docs/_sass/vendor/bourbon/functions/_strip-units.scss +0 -22
  241. data/docs/_sass/vendor/bourbon/functions/_tint.scss +0 -24
  242. data/docs/_sass/vendor/bourbon/functions/_transition-property-name.scss +0 -37
  243. data/docs/_sass/vendor/bourbon/functions/_unpack.scss +0 -32
  244. data/docs/_sass/vendor/bourbon/helpers/_convert-units.scss +0 -26
  245. data/docs/_sass/vendor/bourbon/helpers/_directional-values.scss +0 -108
  246. data/docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss +0 -53
  247. data/docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss +0 -24
  248. data/docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss +0 -35
  249. data/docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss +0 -51
  250. data/docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss +0 -77
  251. data/docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss +0 -41
  252. data/docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss +0 -74
  253. data/docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss +0 -55
  254. data/docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss +0 -28
  255. data/docs/_sass/vendor/bourbon/helpers/_render-gradients.scss +0 -31
  256. data/docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss +0 -15
  257. data/docs/_sass/vendor/bourbon/helpers/_str-to-num.scss +0 -55
  258. data/docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss +0 -7
  259. data/docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss +0 -8
  260. data/docs/_sass/vendor/bourbon/settings/_prefixer.scss +0 -9
  261. data/docs/_sass/vendor/bourbon/settings/_px-to-em.scss +0 -1
  262. data/docs/_sass/vendor/neat/_neat-helpers.scss +0 -11
  263. data/docs/_sass/vendor/neat/_neat.scss +0 -23
  264. data/docs/_sass/vendor/neat/functions/_new-breakpoint.scss +0 -49
  265. data/docs/_sass/vendor/neat/functions/_private.scss +0 -114
  266. data/docs/_sass/vendor/neat/grid/_box-sizing.scss +0 -15
  267. data/docs/_sass/vendor/neat/grid/_direction-context.scss +0 -33
  268. data/docs/_sass/vendor/neat/grid/_display-context.scss +0 -28
  269. data/docs/_sass/vendor/neat/grid/_fill-parent.scss +0 -22
  270. data/docs/_sass/vendor/neat/grid/_media.scss +0 -92
  271. data/docs/_sass/vendor/neat/grid/_omega.scss +0 -87
  272. data/docs/_sass/vendor/neat/grid/_outer-container.scss +0 -34
  273. data/docs/_sass/vendor/neat/grid/_pad.scss +0 -25
  274. data/docs/_sass/vendor/neat/grid/_private.scss +0 -35
  275. data/docs/_sass/vendor/neat/grid/_row.scss +0 -52
  276. data/docs/_sass/vendor/neat/grid/_shift.scss +0 -50
  277. data/docs/_sass/vendor/neat/grid/_span-columns.scss +0 -94
  278. data/docs/_sass/vendor/neat/grid/_to-deprecate.scss +0 -97
  279. data/docs/_sass/vendor/neat/grid/_visual-grid.scss +0 -42
  280. data/docs/_sass/vendor/neat/mixins/_clearfix.scss +0 -25
  281. data/docs/_sass/vendor/neat/settings/_disable-warnings.scss +0 -13
  282. data/docs/_sass/vendor/neat/settings/_grid.scss +0 -51
  283. data/docs/_sass/vendor/neat/settings/_visual-grid.scss +0 -27
  284. data/docs/_sass/vendor/normalize-3.0.2.scss +0 -427
  285. data/docs/_sass/vendor/pygments.scss +0 -356
  286. data/docs/automating_browsers/capybara.md +0 -70
  287. data/docs/css/screen.scss +0 -7
  288. data/docs/guides/cli.md +0 -52
  289. data/docs/guides/frontiers.md +0 -93
  290. data/docs/guides/halting.md +0 -23
  291. data/docs/guides/job_queues.md +0 -26
  292. data/docs/guides/locals.md +0 -36
  293. data/docs/guides/logging.md +0 -22
  294. data/docs/guides/page_objects.md +0 -67
  295. data/docs/guides/peeking.md +0 -46
  296. data/docs/guides/selenium_capybara.md +0 -100
  297. data/docs/guides/tutorial.md +0 -452
  298. data/docs/js/navigation.js +0 -11
  299. data/docs/misc/contributing.md +0 -20
  300. data/docs/misc/testing.md +0 -11
  301. data/docs/recipes/authentication.md +0 -23
  302. data/docs/recipes/csv.md +0 -29
  303. data/docs/recipes/javascript.md +0 -20
  304. data/docs/recipes/multiple_uris.md +0 -18
  305. data/docs/recipes/screenshots.md +0 -20
  306. data/docs/routing/custom_rules.md +0 -16
  307. data/docs/routing/filetypes_rules.md +0 -21
  308. data/docs/routing/host_rules.md +0 -24
  309. data/docs/routing/path_rules.md +0 -33
  310. data/docs/routing/protocol_rules.md +0 -17
  311. data/docs/routing/query_rules.md +0 -69
  312. data/docs/routing/routes.md +0 -96
  313. data/docs/routing/uri_rules.md +0 -18
  314. data/examples/collect_github_issues.rb +0 -65
  315. data/examples/find_foobar_on_wikipedia.rb +0 -23
  316. data/lib/wayfarer/configuration.rb +0 -86
  317. data/lib/wayfarer/crawl.rb +0 -79
  318. data/lib/wayfarer/crawl_observer.rb +0 -103
  319. data/lib/wayfarer/dispatcher.rb +0 -104
  320. data/lib/wayfarer/finders.rb +0 -61
  321. data/lib/wayfarer/frontiers/frontier.rb +0 -79
  322. data/lib/wayfarer/frontiers/memory_bloomfilter.rb +0 -32
  323. data/lib/wayfarer/frontiers/memory_frontier.rb +0 -76
  324. data/lib/wayfarer/frontiers/memory_trie_frontier.rb +0 -39
  325. data/lib/wayfarer/frontiers/normalize_uris.rb +0 -48
  326. data/lib/wayfarer/frontiers/redis_bloomfilter.rb +0 -34
  327. data/lib/wayfarer/frontiers/redis_frontier.rb +0 -83
  328. data/lib/wayfarer/http_adapters/adapter_pool.rb +0 -62
  329. data/lib/wayfarer/http_adapters/net_http_adapter.rb +0 -77
  330. data/lib/wayfarer/http_adapters/selenium_adapter.rb +0 -80
  331. data/lib/wayfarer/job.rb +0 -211
  332. data/lib/wayfarer/locals.rb +0 -40
  333. data/lib/wayfarer/parsers/json_parser.rb +0 -20
  334. data/lib/wayfarer/parsers/xml_parser.rb +0 -27
  335. data/lib/wayfarer/processor.rb +0 -103
  336. data/lib/wayfarer/routing/custom_rule.rb +0 -21
  337. data/lib/wayfarer/routing/filetypes_rule.rb +0 -20
  338. data/lib/wayfarer/routing/host_rule.rb +0 -19
  339. data/lib/wayfarer/routing/path_rule.rb +0 -54
  340. data/lib/wayfarer/routing/protocol_rule.rb +0 -21
  341. data/lib/wayfarer/routing/query_rule.rb +0 -59
  342. data/lib/wayfarer/routing/rule.rb +0 -114
  343. data/lib/wayfarer/routing/uri_rule.rb +0 -21
  344. data/spec/configuration_spec.rb +0 -26
  345. data/spec/crawl_spec.rb +0 -48
  346. data/spec/finders_spec.rb +0 -49
  347. data/spec/frontiers/memory_bloomfilter_spec.rb +0 -6
  348. data/spec/frontiers/memory_frontier_spec.rb +0 -6
  349. data/spec/frontiers/memory_trie_frontier_spec.rb +0 -6
  350. data/spec/frontiers/normalize_uris_spec.rb +0 -59
  351. data/spec/frontiers/redis_bloomfilter_spec.rb +0 -6
  352. data/spec/frontiers/redis_frontier_spec.rb +0 -6
  353. data/spec/http_adapters/adapter_pool_spec.rb +0 -33
  354. data/spec/http_adapters/net_http_adapter_spec.rb +0 -83
  355. data/spec/http_adapters/selenium_adapter_spec.rb +0 -53
  356. data/spec/integration/callbacks_spec.rb +0 -42
  357. data/spec/integration/locals_spec.rb +0 -106
  358. data/spec/integration/peeking_spec.rb +0 -61
  359. data/spec/job_spec.rb +0 -122
  360. data/spec/processor_spec.rb +0 -31
  361. data/spec/routing/custom_rule_spec.rb +0 -26
  362. data/spec/routing/host_rule_spec.rb +0 -48
  363. data/spec/routing/path_rule_spec.rb +0 -66
  364. data/spec/routing/protocol_rule_spec.rb +0 -26
  365. data/spec/routing/query_rule_spec.rb +0 -124
  366. data/spec/routing/rule_spec.rb +0 -251
  367. data/spec/routing/uri_rule_spec.rb +0 -24
  368. data/spec/shared/frontier.rb +0 -96
  369. data/wayfarer-jruby.gemspec +0 -49
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "spec_helpers"
4
+
5
+ describe Wayfarer::Middleware::Chain do
6
+ let(:task) { build(:task) }
7
+ subject(:chain) { described_class.new(middlewares) }
8
+
9
+ describe "#call" do
10
+ context "without middlewares" do
11
+ let(:middlewares) { [] }
12
+
13
+ it "does not raise" do
14
+ expect { chain.call(task) }.not_to raise_error
15
+ end
16
+
17
+ it "yields" do
18
+ expect { |spy| chain.call(task, &spy) }.to yield_control
19
+ end
20
+ end
21
+
22
+ context "when middleware yields" do
23
+ let(:middlewares) { [spy, spy] }
24
+
25
+ it "yields" do
26
+ allow(middlewares.first).to receive(:call).and_yield
27
+ expect(middlewares.last).to receive(:call).with(task).and_yield
28
+ expect { |spy| chain.call(task, &spy) }.to yield_control
29
+ end
30
+ end
31
+
32
+ context "when middleware does not yield" do
33
+ let(:middlewares) { [spy, spy] }
34
+
35
+ it "does not yield" do
36
+ allow(middlewares.first).to receive(:call)
37
+ expect(middlewares.last).not_to receive(:call)
38
+ expect { |spy| chain.call(task, &spy) }.not_to yield_control
39
+ end
40
+ end
41
+
42
+ describe "Immutability" do
43
+ let(:middlewares) { [spy, spy] }
44
+
45
+ it "does not alter its middleware" do
46
+ allow(middlewares.first).to receive(:call).and_yield
47
+ allow(middlewares.second).to receive(:call).and_yield
48
+ expect { chain.call(task) }.not_to(change { chain.middlewares })
49
+ end
50
+ end
51
+
52
+ describe "Return value" do
53
+ let(:middlewares) do
54
+ [build(:middleware, receiver: lambda do |task, &block|
55
+ task.metadata.foobar = 42
56
+ block.call
57
+ end),
58
+ build(:middleware, receiver: lambda do |task|
59
+ task.metadata.foobar *= 1337
60
+ end)]
61
+ end
62
+
63
+ it "is the last middleware's return value" do
64
+ expect(chain.call(task)).to be(56_154)
65
+ end
66
+ end
67
+
68
+ describe "Metadata" do
69
+ let(:first) do
70
+ build(:middleware, receiver: lambda do |task, &block|
71
+ task.metadata.foobar = 42
72
+ block.call
73
+ task.metadata.barqux = 1337
74
+ end)
75
+ end
76
+
77
+ let(:last) do
78
+ build(:middleware, receiver: lambda do |task|
79
+ raise unless task.metadata.foobar == 42
80
+ raise if task.metadata.barqux
81
+ end)
82
+ end
83
+
84
+ let(:middlewares) { [first, last] }
85
+
86
+ it "is accessible across middleware" do
87
+ expect {
88
+ chain.call(task)
89
+ }.not_to raise_error
90
+
91
+ expect(task.metadata.foobar).to be(42)
92
+ expect(task.metadata.barqux).to be(1337)
93
+ end
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "spec_helpers"
4
+
5
+ describe Wayfarer::Middleware::Dedup, redis: true do
6
+ let(:task) { build(:task) }
7
+ subject(:dedup) { Wayfarer::Middleware::Dedup.new }
8
+
9
+ before { task.metadata.staged_urls = SortedSet.new }
10
+
11
+ describe "#call" do
12
+ context "without staged URLs" do
13
+ it "does not raise" do
14
+ expect { dedup.call(task) }.not_to raise_error
15
+ end
16
+ end
17
+
18
+ context "with unseen URL" do
19
+ it "marks the URL as seen" do
20
+ expect {
21
+ dedup.call(task)
22
+ }.to change { task.barrier.seen?(task.url) }.to(true)
23
+ end
24
+
25
+ it "yields" do
26
+ expect { |spy| dedup.call(task, &spy) }.to yield_control
27
+ end
28
+ end
29
+
30
+ context "with seen URL" do
31
+ before { task.barrier.seen?(task.url) }
32
+
33
+ it "does not yield" do
34
+ expect { |spy| dedup.call(task, &spy) }.not_to yield_control
35
+ end
36
+ end
37
+
38
+ context "with exception raised" do
39
+ it "does not mark the URL as seen" do
40
+ begin
41
+ dedup.call(task) { raise }
42
+ rescue StandardError
43
+ nil
44
+ end
45
+ expect(task.barrier.seen?(task.url)).to be(false)
46
+ end
47
+
48
+ it "re-raises the exception" do
49
+ expect {
50
+ dedup.call(task) { raise }
51
+ }.to raise_error(RuntimeError)
52
+ end
53
+ end
54
+
55
+ describe "staged URL filtering" do
56
+ let(:seen_urls) { %w[https://yahoo.com https://google.com] }
57
+ let(:unseen_urls) { %w[https://w3c.org https://nasa.gov] }
58
+
59
+ before do
60
+ seen_urls.each do |url|
61
+ task.barrier.seen?(url)
62
+ end
63
+
64
+ [*seen_urls, *unseen_urls].each do |url|
65
+ task.metadata.staged_urls.add(url)
66
+ end
67
+ end
68
+
69
+ it "filters seen staged URLs" do
70
+ expect {
71
+ dedup.call(task)
72
+ }.to change { task.metadata.staged_urls }.to(SortedSet.new(unseen_urls))
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,100 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "spec_helpers"
4
+
5
+ describe Wayfarer::Middleware::Fetch do
6
+ let(:klass) { Class.new(Wayfarer::Base) }
7
+ let(:job) { klass.new }
8
+ let(:task) { build(:task) }
9
+ let(:context) { double(instance: instance) }
10
+ let(:instance) { spy }
11
+ subject(:fetch) { Wayfarer::Middleware::Fetch.new }
12
+
13
+ before do
14
+ task.job = job
15
+ allow(fetch.pool).to receive(:with).and_yield(context)
16
+ allow(context).to receive(:fetch)
17
+ end
18
+
19
+ describe "#call" do
20
+ it "assigns the task" do
21
+ expect {
22
+ fetch.call(task)
23
+ }.to change { fetch.task }.to(task)
24
+ end
25
+
26
+ it "retrieves the page" do
27
+ expect(context).to receive(:fetch).with(task.url)
28
+ fetch.call(task)
29
+ end
30
+
31
+ it "runs callbacks" do
32
+ expect { |spy|
33
+ klass.before_fetch(&spy)
34
+ fetch.call(task)
35
+ }.to yield_control
36
+ end
37
+
38
+ context "when encountering an HTTP redirect" do
39
+ let(:redirect_url) { "https://redirect.com" }
40
+
41
+ let(:result) do
42
+ Wayfarer::Networking::Result::Redirect.new(redirect_url)
43
+ end
44
+
45
+ before do
46
+ task.metadata.staged_urls = Set.new
47
+ allow(context).to receive(:fetch).with(task.url).and_return(result)
48
+ end
49
+
50
+ it "stages the redirect URL" do
51
+ expect {
52
+ fetch.call(task)
53
+ }.to change { task.metadata.staged_urls }.to([redirect_url])
54
+ end
55
+
56
+ it "does not yield" do
57
+ expect { |spy| fetch.call(task, &spy) }.not_to yield_control
58
+ end
59
+ end
60
+
61
+ context "when retrieval succeeds" do
62
+ let(:page) { build(:page) }
63
+ let(:result) { Wayfarer::Networking::Result::Success.new(page) }
64
+
65
+ before do
66
+ allow(context).to receive(:fetch).with(task.url).and_return(result)
67
+ end
68
+
69
+ it "assigns the page" do
70
+ expect {
71
+ fetch.call(task)
72
+ }.to change { task.metadata.page }.to(page)
73
+ end
74
+
75
+ it "assigns the instance" do
76
+ expect {
77
+ fetch.call(task)
78
+ }.to change { task.metadata.agent }.to(instance)
79
+ end
80
+
81
+ it "yields" do
82
+ expect { |spy| fetch.call(task, &spy) }.to yield_control
83
+ end
84
+ end
85
+ end
86
+
87
+ describe described_class::API do
88
+ let(:klass) do
89
+ Class.new(Wayfarer::Base) do
90
+ route do |host, path|
91
+ to :index, host: host, path: path
92
+ end
93
+
94
+ steer do |task|
95
+ [task.batch, "/foobar"]
96
+ end
97
+ end
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "spec_helpers"
4
+
5
+ describe Wayfarer::Middleware::Normalize do
6
+ let(:task) { build(:task) }
7
+ subject(:normalize) { Wayfarer::Middleware::Normalize.new }
8
+
9
+ describe "#call" do
10
+ let(:urls) do
11
+ ["http://example.com/products?product_id=123",
12
+ "HTTP://EXAMPLE.COM/products/?product_id=123",
13
+ "http://example.com/products/?product_id=123",
14
+ "http://example.com/foo/../products?product_id=123"]
15
+ end
16
+
17
+ before { task.metadata.staged_urls = SortedSet.new(urls) }
18
+
19
+ it "yields" do
20
+ expect { |spy| normalize.call(task, &spy) }.to yield_control
21
+ end
22
+
23
+ it "normalizes and compacts URLs" do
24
+ normalize.call(task)
25
+ expect(task.metadata.staged_urls).to eq(SortedSet[urls.first])
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,80 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "spec_helpers"
4
+
5
+ describe Wayfarer::Middleware::Router do
6
+ let(:task) { build(:task, url: url) }
7
+ subject(:router) { Wayfarer::Middleware::Router.new }
8
+
9
+ before do
10
+ task.job = klass.new
11
+ allow(task.job).to receive(:arguments).and_return([task])
12
+ end
13
+
14
+ describe "#call" do
15
+ let(:klass) do
16
+ Class.new(Wayfarer::Base) do
17
+ route do
18
+ to :index, host: "example.com", path: "/foobar"
19
+ end
20
+ end
21
+ end
22
+
23
+ context "when URL matches a route" do
24
+ let(:url) { "https://example.com/foobar" }
25
+
26
+ it "yields" do
27
+ expect { |spy| router.call(task, &spy) }.to yield_control
28
+ end
29
+
30
+ it "assigns metadata" do
31
+ router.call(task)
32
+ expect(task.metadata.action).to be(:index)
33
+ expect(task.metadata.params).to eq({})
34
+ end
35
+
36
+ describe "params" do
37
+ it "is a HashWithIndifferentAccess" do
38
+ router.call(task)
39
+ expect(task.metadata.params).to be_a(ActiveSupport::HashWithIndifferentAccess)
40
+ end
41
+ end
42
+ end
43
+
44
+ context "when URL does not match a route" do
45
+ let(:url) { "https://yahoo.com/foobar" }
46
+
47
+ it "does not yield" do
48
+ expect { |spy| router.call(task, &spy) }.not_to yield_control
49
+ end
50
+
51
+ it "does not assign metadata" do
52
+ router.call(task) {}
53
+ expect(task.metadata.action).to be(nil)
54
+ expect(task.metadata.params).to be(nil)
55
+ end
56
+ end
57
+ end
58
+
59
+ describe described_class::API do
60
+ let(:klass) do
61
+ Class.new(Wayfarer::Base) do
62
+ route do |host, path|
63
+ to :index, host: host, path: path
64
+ end
65
+
66
+ steer do |task|
67
+ [task.batch, "/foobar"]
68
+ end
69
+ end
70
+ end
71
+
72
+ let(:task) { build(:task, batch: "example.com", url: "https://example.com/foobar") }
73
+
74
+ it "passes arguments" do
75
+ router.call(task)
76
+ expect(task.metadata.action).to be(:index)
77
+ expect(task.metadata.params).to eq({})
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "spec_helpers"
4
+
5
+ describe Wayfarer::Middleware::Stage do
6
+ include ActiveJob::TestHelper
7
+
8
+ let(:task) { build(:task) }
9
+ subject(:stage) { Wayfarer::Middleware::Stage.new }
10
+
11
+ describe "#call" do
12
+ it "assigns staged URLs" do
13
+ stage.call(task) do
14
+ expect(task.metadata.staged_urls).to eq(SortedSet.new)
15
+ end
16
+ end
17
+
18
+ it "yields" do
19
+ expect { |spy| stage.call(task, &spy) }.to yield_control
20
+ end
21
+
22
+ describe "Task queuing" do
23
+ let(:klass) { Class.new(Wayfarer::Base) }
24
+ let(:urls) { %w[https://example.com https://w3c.org] }
25
+
26
+ before { task.job = klass.new }
27
+
28
+ it "enqueues tasks" do
29
+ expect(klass).to receive(:crawl_later).with(urls.first, batch: task.batch)
30
+ expect(klass).to receive(:crawl_later).with(urls.second, batch: task.batch)
31
+
32
+ stage.call(task) do
33
+ extend Wayfarer::Middleware::Stage::API
34
+ stage(urls)
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,117 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "spec_helpers"
4
+
5
+ describe Wayfarer::Middleware::Worker do
6
+ let(:task) { build(:task) }
7
+
8
+ let(:job) do
9
+ Class.new(Wayfarer::Base) do
10
+ def index
11
+ :foobar
12
+ end
13
+ end
14
+ end
15
+
16
+ before do
17
+ task.metadata.action = :index
18
+ allow_any_instance_of(job).to receive(:arguments).and_return([task])
19
+ end
20
+
21
+ describe "::router" do
22
+ it "returns a router" do
23
+ expect(job.router).to be_a(Wayfarer::Routing::Router)
24
+ end
25
+ end
26
+
27
+ describe "#call" do
28
+ it "calls the action method" do
29
+ worker = job.new
30
+ expect(worker).to receive(:index)
31
+ worker.call(task)
32
+ end
33
+
34
+ it "yields" do
35
+ expect { |spy| job.new.call(task, &spy) }.to yield_control
36
+ end
37
+ end
38
+
39
+ describe "#chain" do
40
+ it "returns its middleware chain" do
41
+ worker = job.new
42
+ expect(worker.chain.middlewares).to eq([*Wayfarer.middleware, worker])
43
+ end
44
+ end
45
+
46
+ describe "#redis" do
47
+ it "yields a Redis connection" do
48
+ job.new.redis do |conn|
49
+ expect(conn).to be_a(Redis)
50
+ end
51
+ end
52
+ end
53
+
54
+ describe "#params" do
55
+ it "delegates to task metadata" do
56
+ worker = job.new
57
+ expect(worker.task.metadata).to receive(:params)
58
+ worker.params
59
+ end
60
+ end
61
+
62
+ describe "#agent" do
63
+ it "delegates to task metadata" do
64
+ worker = job.new
65
+ expect(worker.task.metadata).to receive(:agent)
66
+ worker.agent
67
+ end
68
+ end
69
+
70
+ describe "#page" do
71
+ let(:worker) { job.new }
72
+ let(:page) { Object.new }
73
+
74
+ it "returns the page" do
75
+ task.metadata.page = page
76
+ expect(worker.page).to be(page)
77
+ end
78
+
79
+ context "with live keyword" do
80
+ let(:agent) { spy }
81
+ let(:result) { Wayfarer::Networking::Result::Success.new(page) }
82
+
83
+ before { task.metadata.agent = agent }
84
+
85
+ context "when agent returns Result" do
86
+ before { allow(agent).to receive(:live).and_return(result) }
87
+
88
+ it "returns the page" do
89
+ expect(worker.page(live: true)).to be(page)
90
+ end
91
+
92
+ it "replaces the existing page" do
93
+ expect {
94
+ worker.page(live: true)
95
+ }.to change { task.metadata.page }.to(page)
96
+ end
97
+ end
98
+
99
+ context "when agent returns nil" do
100
+ before do
101
+ task.metadata.page = page
102
+ allow(agent).to receive(:live).and_return(nil)
103
+ end
104
+
105
+ it "returns the existing page" do
106
+ expect(worker.page(live: true)).to be(page)
107
+ end
108
+
109
+ it "does not replace the existing page" do
110
+ expect {
111
+ worker.page(live: true)
112
+ }.not_to(change { task.metadata.page })
113
+ end
114
+ end
115
+ end
116
+ end
117
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "spec_helpers"
4
+ require_relative "strategy"
5
+
6
+ describe Wayfarer::Networking::Capybara, ferrum: true do
7
+ include_examples "Network strategy", strategy: described_class,
8
+ browser: true,
9
+ request_headers: false,
10
+ response_headers: true,
11
+ status_code: true
12
+ end
@@ -0,0 +1,127 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "spec_helpers"
4
+
5
+ describe Wayfarer::Networking::Context do
6
+ let(:instance) { Object.new }
7
+ let(:strategy) { spy }
8
+ let(:renewing_error) { Class.new(StandardError) }
9
+ subject(:context) { Wayfarer::Networking::Context.new(strategy) }
10
+
11
+ before do
12
+ allow(strategy).to receive(:renew_on).and_return([renewing_error])
13
+ allow(strategy).to receive(:create).and_return(instance)
14
+ end
15
+
16
+ describe "#new" do
17
+ it "assigns the strategy" do
18
+ expect(context.strategy).to be(strategy)
19
+ end
20
+
21
+ it "assigns the instance" do
22
+ expect(context.strategy).to be(strategy)
23
+ end
24
+ end
25
+
26
+ describe "#fetch" do
27
+ let(:url) { Object.new }
28
+
29
+ it "fetches" do
30
+ expect(strategy).to receive(:fetch).with(instance, url)
31
+ context.fetch(url)
32
+ end
33
+
34
+ context "with renewing exception raised" do
35
+ before do
36
+ allow(strategy).to receive(:fetch).and_raise(renewing_error)
37
+ end
38
+
39
+ it "renews and reraises" do
40
+ expect(context).to receive(:renew)
41
+
42
+ expect {
43
+ context.fetch(url)
44
+ }.to raise_error(renewing_error)
45
+ end
46
+ end
47
+
48
+ context "with non-renewing exception raised" do
49
+ before do
50
+ allow(strategy).to receive(:fetch).and_raise(StandardError)
51
+ end
52
+
53
+ it "reraises" do
54
+ expect(context).not_to receive(:renew)
55
+
56
+ expect {
57
+ context.fetch(url)
58
+ }.to raise_error(StandardError)
59
+ end
60
+ end
61
+ end
62
+
63
+ describe "#live" do
64
+ let(:page) { Object.new }
65
+
66
+ it "calls live" do
67
+ expect(strategy).to receive(:live).with(instance)
68
+ context.live
69
+ end
70
+
71
+ context "with renewing exception raised" do
72
+ before do
73
+ allow(strategy).to receive(:live).and_raise(renewing_error)
74
+ end
75
+
76
+ it "renews and reraises" do
77
+ expect(context).to receive(:renew)
78
+
79
+ expect {
80
+ context.live
81
+ }.to raise_error(renewing_error)
82
+ end
83
+
84
+ context "when renewing raises" do
85
+ let(:other_error) { Class.new(StandardError) }
86
+
87
+ before do
88
+ allow(context).to receive(:renew).and_raise(other_error)
89
+ end
90
+
91
+ it "reraises the original exception" do
92
+ expect {
93
+ context.live
94
+ }.to raise_error(renewing_error)
95
+ end
96
+ end
97
+ end
98
+
99
+ context "with non-renewing exception raised" do
100
+ before do
101
+ allow(strategy).to receive(:live).and_raise(StandardError)
102
+ end
103
+
104
+ it "reraises" do
105
+ expect(context).not_to receive(:renew)
106
+
107
+ expect {
108
+ context.live(page)
109
+ }.to raise_error(StandardError)
110
+ end
111
+ end
112
+ end
113
+
114
+ describe "#renew" do
115
+ it "destroys" do
116
+ expect(strategy).to receive(:destroy).with(instance)
117
+ context.renew
118
+ end
119
+ end
120
+
121
+ describe "#instance" do
122
+ it "creates an instance" do
123
+ expect(strategy).to receive(:create).and_return(instance)
124
+ expect(context.instance).to be(instance)
125
+ end
126
+ end
127
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "spec_helpers"
4
+ require_relative "strategy"
5
+
6
+ describe Wayfarer::Networking::Ferrum, ferrum: true do
7
+ include_examples "Network strategy", strategy: described_class,
8
+ browser: true,
9
+ request_headers: true,
10
+ response_headers: true,
11
+ status_code: true
12
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "spec_helpers"
4
+ require_relative "strategy"
5
+
6
+ describe Wayfarer::Networking::HTTP do
7
+ include_examples "Network strategy", strategy: described_class,
8
+ browser: false,
9
+ request_headers: true,
10
+ response_headers: true,
11
+ status_code: true
12
+ end