wayfarer 0.0.3 → 0.4.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (369) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/ci.yaml +32 -0
  3. data/.gitignore +3 -4
  4. data/.rubocop.yml +25 -9
  5. data/.ruby-version +1 -1
  6. data/Dockerfile +5 -0
  7. data/Gemfile +1 -7
  8. data/Gemfile.lock +221 -0
  9. data/RELEASING.md +17 -0
  10. data/Rakefile +38 -90
  11. data/bin/wayfarer +1 -111
  12. data/docker-compose.yml +32 -0
  13. data/docs/cookbook/batch_routing.md +22 -0
  14. data/docs/cookbook/consent_screen.md +36 -0
  15. data/docs/cookbook/executing_javascript.md +41 -0
  16. data/docs/cookbook/querying_html.md +42 -0
  17. data/docs/cookbook/screenshots.md +27 -0
  18. data/docs/cookbook/user_agent.md +7 -0
  19. data/docs/guides/browser_automation/capybara.md +69 -0
  20. data/docs/guides/browser_automation/custom_adapters.md +100 -0
  21. data/docs/guides/browser_automation/ferrum.md +39 -0
  22. data/docs/guides/browser_automation/selenium.md +63 -0
  23. data/docs/guides/callbacks.md +131 -31
  24. data/docs/guides/configuration.md +24 -169
  25. data/docs/guides/debugging.md +17 -0
  26. data/docs/guides/error_handling.md +30 -45
  27. data/docs/guides/jobs.md +101 -0
  28. data/docs/guides/navigation.md +73 -0
  29. data/docs/guides/networking.md +94 -0
  30. data/docs/guides/pages.md +52 -0
  31. data/docs/guides/performance.md +130 -0
  32. data/docs/guides/reliability.md +41 -0
  33. data/docs/guides/routing/steering.md +30 -0
  34. data/docs/guides/tasks.md +14 -0
  35. data/docs/index.md +40 -66
  36. data/docs/reference/api/base.md +48 -0
  37. data/docs/reference/api/route.md +182 -0
  38. data/docs/reference/cli.md +61 -0
  39. data/docs/reference/configuration_keys.md +42 -0
  40. data/docs/reference/environment_variables.md +83 -0
  41. data/lib/wayfarer/base.rb +50 -0
  42. data/lib/wayfarer/callbacks.rb +71 -0
  43. data/lib/wayfarer/cli/base.rb +27 -0
  44. data/lib/wayfarer/cli/generate.rb +17 -0
  45. data/lib/wayfarer/cli/job.rb +60 -0
  46. data/lib/wayfarer/cli/route.rb +29 -0
  47. data/lib/wayfarer/cli/route_printer.rb +116 -0
  48. data/lib/wayfarer/cli/runner.rb +34 -0
  49. data/lib/wayfarer/cli/templates/Gemfile.tt +5 -0
  50. data/lib/wayfarer/cli/templates/job.rb.tt +10 -0
  51. data/lib/wayfarer/config/capybara.rb +10 -0
  52. data/lib/wayfarer/config/ferrum.rb +11 -0
  53. data/lib/wayfarer/config/networking.rb +26 -0
  54. data/lib/wayfarer/config/redis.rb +14 -0
  55. data/lib/wayfarer/config/root.rb +11 -0
  56. data/lib/wayfarer/config/selenium.rb +21 -0
  57. data/lib/wayfarer/config/strconv.rb +45 -0
  58. data/lib/wayfarer/config/struct.rb +72 -0
  59. data/lib/wayfarer/gc.rb +15 -0
  60. data/lib/wayfarer/middleware/chain.rb +19 -0
  61. data/lib/wayfarer/middleware/dedup.rb +25 -0
  62. data/lib/wayfarer/middleware/fetch.rb +47 -0
  63. data/lib/wayfarer/middleware/normalize.rb +25 -0
  64. data/lib/wayfarer/middleware/router.rb +53 -0
  65. data/lib/wayfarer/middleware/stage.rb +23 -0
  66. data/lib/wayfarer/middleware/worker.rb +30 -0
  67. data/lib/wayfarer/networking/capybara.rb +28 -0
  68. data/lib/wayfarer/networking/context.rb +36 -0
  69. data/lib/wayfarer/networking/ferrum.rb +35 -0
  70. data/lib/wayfarer/networking/http.rb +34 -0
  71. data/lib/wayfarer/networking/pool.rb +40 -0
  72. data/lib/wayfarer/networking/result.rb +18 -0
  73. data/lib/wayfarer/networking/selenium.rb +43 -0
  74. data/lib/wayfarer/networking/strategy.rb +38 -0
  75. data/lib/wayfarer/page.rb +17 -74
  76. data/lib/wayfarer/parsing/json.rb +17 -0
  77. data/lib/wayfarer/parsing/xml.rb +17 -0
  78. data/lib/wayfarer/redis/.#barrier.rb +1 -0
  79. data/lib/wayfarer/redis/barrier.rb +36 -0
  80. data/lib/wayfarer/redis/connection.rb +13 -0
  81. data/lib/wayfarer/redis/counter.rb +29 -0
  82. data/lib/wayfarer/redis/pool.rb +20 -0
  83. data/lib/wayfarer/redis/version.rb +19 -0
  84. data/lib/wayfarer/routing/dsl.rb +57 -0
  85. data/lib/wayfarer/routing/matchers/custom.rb +25 -0
  86. data/lib/wayfarer/routing/matchers/host.rb +19 -0
  87. data/lib/wayfarer/routing/matchers/path.rb +49 -0
  88. data/lib/wayfarer/routing/matchers/query.rb +63 -0
  89. data/lib/wayfarer/routing/matchers/scheme.rb +17 -0
  90. data/lib/wayfarer/routing/matchers/suffix.rb +17 -0
  91. data/lib/wayfarer/routing/matchers/url.rb +17 -0
  92. data/lib/wayfarer/routing/path_finder.rb +46 -0
  93. data/lib/wayfarer/routing/result.rb +15 -0
  94. data/lib/wayfarer/routing/root_route.rb +7 -0
  95. data/lib/wayfarer/routing/route.rb +47 -0
  96. data/lib/wayfarer/routing/router.rb +10 -54
  97. data/lib/wayfarer/routing/target_route.rb +7 -0
  98. data/lib/wayfarer/serializer.rb +17 -0
  99. data/lib/wayfarer/stringify.rb +47 -0
  100. data/lib/wayfarer/task.rb +34 -0
  101. data/lib/wayfarer.rb +48 -57
  102. data/mkdocs.yml +47 -0
  103. data/requirements.txt +1 -0
  104. data/spec/base_spec.rb +233 -0
  105. data/spec/callbacks_spec.rb +102 -0
  106. data/spec/cli/generate_spec.rb +39 -0
  107. data/spec/cli/job_spec.rb +74 -0
  108. data/spec/cli/version_spec.rb +13 -0
  109. data/spec/config/capybara_spec.rb +18 -0
  110. data/spec/config/ferrum_spec.rb +24 -0
  111. data/spec/config/networking_spec.rb +73 -0
  112. data/spec/config/redis_spec.rb +32 -0
  113. data/spec/config/root_spec.rb +31 -0
  114. data/spec/config/selenium_spec.rb +56 -0
  115. data/spec/config/strconv_spec.rb +58 -0
  116. data/spec/config/struct_spec.rb +66 -0
  117. data/spec/factories/middleware.rb +15 -0
  118. data/spec/factories/page.rb +78 -0
  119. data/spec/factories/task.rb +12 -0
  120. data/spec/fixtures/dummy_job.rb +7 -0
  121. data/spec/gc_spec.rb +63 -0
  122. data/spec/middleware/chain_spec.rb +96 -0
  123. data/spec/middleware/dedup_spec.rb +76 -0
  124. data/spec/middleware/fetch_spec.rb +100 -0
  125. data/spec/middleware/normalize_spec.rb +28 -0
  126. data/spec/middleware/router_spec.rb +80 -0
  127. data/spec/middleware/stage_spec.rb +39 -0
  128. data/spec/middleware/worker_spec.rb +117 -0
  129. data/spec/networking/capybara_spec.rb +12 -0
  130. data/spec/networking/context_spec.rb +127 -0
  131. data/spec/networking/ferrum_spec.rb +12 -0
  132. data/spec/networking/http_spec.rb +12 -0
  133. data/spec/networking/pool_spec.rb +67 -0
  134. data/spec/networking/selenium_spec.rb +12 -0
  135. data/spec/networking/strategy.rb +170 -0
  136. data/spec/page_spec.rb +21 -12
  137. data/spec/{parsers/json_parser_spec.rb → parsing/json_spec.rb} +5 -4
  138. data/spec/{parsers/xml_parser_spec.rb → parsing/xml_spec.rb} +3 -2
  139. data/spec/redis/barrier_spec.rb +78 -0
  140. data/spec/redis/counter_spec.rb +32 -0
  141. data/spec/redis/pool_spec.rb +18 -0
  142. data/spec/redis/version_spec.rb +13 -0
  143. data/spec/routing/dsl_spec.rb +98 -0
  144. data/spec/routing/integration_spec.rb +110 -0
  145. data/spec/routing/matchers/custom_spec.rb +31 -0
  146. data/spec/routing/matchers/host_spec.rb +49 -0
  147. data/spec/routing/matchers/path_spec.rb +43 -0
  148. data/spec/routing/matchers/query_spec.rb +137 -0
  149. data/spec/routing/matchers/scheme_spec.rb +25 -0
  150. data/spec/routing/{filetypes_rule_spec.rb → matchers/suffix_spec.rb} +14 -13
  151. data/spec/routing/matchers/uri_spec.rb +27 -0
  152. data/spec/routing/path_finder_spec.rb +33 -0
  153. data/spec/routing/root_route_spec.rb +29 -0
  154. data/spec/routing/route_spec.rb +43 -0
  155. data/spec/routing/router_spec.rb +13 -56
  156. data/spec/spec_helpers.rb +73 -38
  157. data/spec/stringify_spec.rb +23 -0
  158. data/{support → spec/support}/static/finders.html +0 -0
  159. data/{support → spec/support}/static/graph/details/a.html +0 -0
  160. data/{support → spec/support}/static/graph/details/b.html +0 -0
  161. data/{support → spec/support}/static/graph/index.html +0 -0
  162. data/{support → spec/support}/static/json/dummy.json +0 -0
  163. data/{support → spec/support}/static/links/links.html +0 -0
  164. data/{support → spec/support}/static/xml/dummy.xml +0 -0
  165. data/{support → spec/support}/test_app.rb +9 -2
  166. data/spec/task_spec.rb +27 -0
  167. data/spec/wayfarer_spec.rb +2 -13
  168. data/wayfarer.gemspec +40 -42
  169. metadata +234 -361
  170. data/.travis.yml +0 -5
  171. data/Changelog.md +0 -10
  172. data/README.md +0 -21
  173. data/benchmark/frontiers.rb +0 -143
  174. data/docs/.gitignore +0 -2
  175. data/docs/_config.yml +0 -15
  176. data/docs/_includes/base.html +0 -7
  177. data/docs/_includes/head.html +0 -10
  178. data/docs/_includes/navigation.html +0 -187
  179. data/docs/_layouts/default.html +0 -42
  180. data/docs/_sass/base.scss +0 -439
  181. data/docs/_sass/variables.scss +0 -24
  182. data/docs/_sass/vendor/bourbon/_bourbon-deprecate.scss +0 -19
  183. data/docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss +0 -425
  184. data/docs/_sass/vendor/bourbon/_bourbon.scss +0 -90
  185. data/docs/_sass/vendor/bourbon/addons/_border-color.scss +0 -29
  186. data/docs/_sass/vendor/bourbon/addons/_border-radius.scss +0 -48
  187. data/docs/_sass/vendor/bourbon/addons/_border-style.scss +0 -28
  188. data/docs/_sass/vendor/bourbon/addons/_border-width.scss +0 -28
  189. data/docs/_sass/vendor/bourbon/addons/_buttons.scss +0 -69
  190. data/docs/_sass/vendor/bourbon/addons/_clearfix.scss +0 -25
  191. data/docs/_sass/vendor/bourbon/addons/_ellipsis.scss +0 -30
  192. data/docs/_sass/vendor/bourbon/addons/_font-stacks.scss +0 -31
  193. data/docs/_sass/vendor/bourbon/addons/_hide-text.scss +0 -27
  194. data/docs/_sass/vendor/bourbon/addons/_margin.scss +0 -29
  195. data/docs/_sass/vendor/bourbon/addons/_padding.scss +0 -29
  196. data/docs/_sass/vendor/bourbon/addons/_position.scss +0 -51
  197. data/docs/_sass/vendor/bourbon/addons/_prefixer.scss +0 -66
  198. data/docs/_sass/vendor/bourbon/addons/_retina-image.scss +0 -27
  199. data/docs/_sass/vendor/bourbon/addons/_size.scss +0 -56
  200. data/docs/_sass/vendor/bourbon/addons/_text-inputs.scss +0 -118
  201. data/docs/_sass/vendor/bourbon/addons/_timing-functions.scss +0 -34
  202. data/docs/_sass/vendor/bourbon/addons/_triangle.scss +0 -63
  203. data/docs/_sass/vendor/bourbon/addons/_word-wrap.scss +0 -29
  204. data/docs/_sass/vendor/bourbon/css3/_animation.scss +0 -61
  205. data/docs/_sass/vendor/bourbon/css3/_appearance.scss +0 -5
  206. data/docs/_sass/vendor/bourbon/css3/_backface-visibility.scss +0 -5
  207. data/docs/_sass/vendor/bourbon/css3/_background-image.scss +0 -44
  208. data/docs/_sass/vendor/bourbon/css3/_background.scss +0 -57
  209. data/docs/_sass/vendor/bourbon/css3/_border-image.scss +0 -61
  210. data/docs/_sass/vendor/bourbon/css3/_calc.scss +0 -6
  211. data/docs/_sass/vendor/bourbon/css3/_columns.scss +0 -67
  212. data/docs/_sass/vendor/bourbon/css3/_filter.scss +0 -6
  213. data/docs/_sass/vendor/bourbon/css3/_flex-box.scss +0 -327
  214. data/docs/_sass/vendor/bourbon/css3/_font-face.scss +0 -29
  215. data/docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss +0 -6
  216. data/docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss +0 -12
  217. data/docs/_sass/vendor/bourbon/css3/_hyphens.scss +0 -6
  218. data/docs/_sass/vendor/bourbon/css3/_image-rendering.scss +0 -15
  219. data/docs/_sass/vendor/bourbon/css3/_keyframes.scss +0 -38
  220. data/docs/_sass/vendor/bourbon/css3/_linear-gradient.scss +0 -40
  221. data/docs/_sass/vendor/bourbon/css3/_perspective.scss +0 -12
  222. data/docs/_sass/vendor/bourbon/css3/_placeholder.scss +0 -10
  223. data/docs/_sass/vendor/bourbon/css3/_radial-gradient.scss +0 -40
  224. data/docs/_sass/vendor/bourbon/css3/_selection.scss +0 -44
  225. data/docs/_sass/vendor/bourbon/css3/_text-decoration.scss +0 -27
  226. data/docs/_sass/vendor/bourbon/css3/_transform.scss +0 -21
  227. data/docs/_sass/vendor/bourbon/css3/_transition.scss +0 -81
  228. data/docs/_sass/vendor/bourbon/css3/_user-select.scss +0 -5
  229. data/docs/_sass/vendor/bourbon/functions/_assign-inputs.scss +0 -16
  230. data/docs/_sass/vendor/bourbon/functions/_contains-falsy.scss +0 -25
  231. data/docs/_sass/vendor/bourbon/functions/_contains.scss +0 -31
  232. data/docs/_sass/vendor/bourbon/functions/_is-length.scss +0 -16
  233. data/docs/_sass/vendor/bourbon/functions/_is-light.scss +0 -26
  234. data/docs/_sass/vendor/bourbon/functions/_is-number.scss +0 -16
  235. data/docs/_sass/vendor/bourbon/functions/_is-size.scss +0 -23
  236. data/docs/_sass/vendor/bourbon/functions/_modular-scale.scss +0 -74
  237. data/docs/_sass/vendor/bourbon/functions/_px-to-em.scss +0 -24
  238. data/docs/_sass/vendor/bourbon/functions/_px-to-rem.scss +0 -26
  239. data/docs/_sass/vendor/bourbon/functions/_shade.scss +0 -24
  240. data/docs/_sass/vendor/bourbon/functions/_strip-units.scss +0 -22
  241. data/docs/_sass/vendor/bourbon/functions/_tint.scss +0 -24
  242. data/docs/_sass/vendor/bourbon/functions/_transition-property-name.scss +0 -37
  243. data/docs/_sass/vendor/bourbon/functions/_unpack.scss +0 -32
  244. data/docs/_sass/vendor/bourbon/helpers/_convert-units.scss +0 -26
  245. data/docs/_sass/vendor/bourbon/helpers/_directional-values.scss +0 -108
  246. data/docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss +0 -53
  247. data/docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss +0 -24
  248. data/docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss +0 -35
  249. data/docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss +0 -51
  250. data/docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss +0 -77
  251. data/docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss +0 -41
  252. data/docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss +0 -74
  253. data/docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss +0 -55
  254. data/docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss +0 -28
  255. data/docs/_sass/vendor/bourbon/helpers/_render-gradients.scss +0 -31
  256. data/docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss +0 -15
  257. data/docs/_sass/vendor/bourbon/helpers/_str-to-num.scss +0 -55
  258. data/docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss +0 -7
  259. data/docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss +0 -8
  260. data/docs/_sass/vendor/bourbon/settings/_prefixer.scss +0 -9
  261. data/docs/_sass/vendor/bourbon/settings/_px-to-em.scss +0 -1
  262. data/docs/_sass/vendor/neat/_neat-helpers.scss +0 -11
  263. data/docs/_sass/vendor/neat/_neat.scss +0 -23
  264. data/docs/_sass/vendor/neat/functions/_new-breakpoint.scss +0 -49
  265. data/docs/_sass/vendor/neat/functions/_private.scss +0 -114
  266. data/docs/_sass/vendor/neat/grid/_box-sizing.scss +0 -15
  267. data/docs/_sass/vendor/neat/grid/_direction-context.scss +0 -33
  268. data/docs/_sass/vendor/neat/grid/_display-context.scss +0 -28
  269. data/docs/_sass/vendor/neat/grid/_fill-parent.scss +0 -22
  270. data/docs/_sass/vendor/neat/grid/_media.scss +0 -92
  271. data/docs/_sass/vendor/neat/grid/_omega.scss +0 -87
  272. data/docs/_sass/vendor/neat/grid/_outer-container.scss +0 -34
  273. data/docs/_sass/vendor/neat/grid/_pad.scss +0 -25
  274. data/docs/_sass/vendor/neat/grid/_private.scss +0 -35
  275. data/docs/_sass/vendor/neat/grid/_row.scss +0 -52
  276. data/docs/_sass/vendor/neat/grid/_shift.scss +0 -50
  277. data/docs/_sass/vendor/neat/grid/_span-columns.scss +0 -94
  278. data/docs/_sass/vendor/neat/grid/_to-deprecate.scss +0 -97
  279. data/docs/_sass/vendor/neat/grid/_visual-grid.scss +0 -42
  280. data/docs/_sass/vendor/neat/mixins/_clearfix.scss +0 -25
  281. data/docs/_sass/vendor/neat/settings/_disable-warnings.scss +0 -13
  282. data/docs/_sass/vendor/neat/settings/_grid.scss +0 -51
  283. data/docs/_sass/vendor/neat/settings/_visual-grid.scss +0 -27
  284. data/docs/_sass/vendor/normalize-3.0.2.scss +0 -427
  285. data/docs/_sass/vendor/pygments.scss +0 -356
  286. data/docs/automating_browsers/capybara.md +0 -70
  287. data/docs/css/screen.scss +0 -7
  288. data/docs/guides/cli.md +0 -52
  289. data/docs/guides/frontiers.md +0 -93
  290. data/docs/guides/halting.md +0 -23
  291. data/docs/guides/job_queues.md +0 -26
  292. data/docs/guides/locals.md +0 -36
  293. data/docs/guides/logging.md +0 -22
  294. data/docs/guides/page_objects.md +0 -67
  295. data/docs/guides/peeking.md +0 -46
  296. data/docs/guides/selenium_capybara.md +0 -100
  297. data/docs/guides/tutorial.md +0 -452
  298. data/docs/js/navigation.js +0 -11
  299. data/docs/misc/contributing.md +0 -20
  300. data/docs/misc/testing.md +0 -11
  301. data/docs/recipes/authentication.md +0 -23
  302. data/docs/recipes/csv.md +0 -29
  303. data/docs/recipes/javascript.md +0 -20
  304. data/docs/recipes/multiple_uris.md +0 -18
  305. data/docs/recipes/screenshots.md +0 -20
  306. data/docs/routing/custom_rules.md +0 -16
  307. data/docs/routing/filetypes_rules.md +0 -21
  308. data/docs/routing/host_rules.md +0 -24
  309. data/docs/routing/path_rules.md +0 -33
  310. data/docs/routing/protocol_rules.md +0 -17
  311. data/docs/routing/query_rules.md +0 -69
  312. data/docs/routing/routes.md +0 -96
  313. data/docs/routing/uri_rules.md +0 -18
  314. data/examples/collect_github_issues.rb +0 -65
  315. data/examples/find_foobar_on_wikipedia.rb +0 -23
  316. data/lib/wayfarer/configuration.rb +0 -86
  317. data/lib/wayfarer/crawl.rb +0 -79
  318. data/lib/wayfarer/crawl_observer.rb +0 -103
  319. data/lib/wayfarer/dispatcher.rb +0 -104
  320. data/lib/wayfarer/finders.rb +0 -61
  321. data/lib/wayfarer/frontiers/frontier.rb +0 -79
  322. data/lib/wayfarer/frontiers/memory_bloomfilter.rb +0 -32
  323. data/lib/wayfarer/frontiers/memory_frontier.rb +0 -76
  324. data/lib/wayfarer/frontiers/memory_trie_frontier.rb +0 -39
  325. data/lib/wayfarer/frontiers/normalize_uris.rb +0 -48
  326. data/lib/wayfarer/frontiers/redis_bloomfilter.rb +0 -34
  327. data/lib/wayfarer/frontiers/redis_frontier.rb +0 -83
  328. data/lib/wayfarer/http_adapters/adapter_pool.rb +0 -62
  329. data/lib/wayfarer/http_adapters/net_http_adapter.rb +0 -77
  330. data/lib/wayfarer/http_adapters/selenium_adapter.rb +0 -80
  331. data/lib/wayfarer/job.rb +0 -211
  332. data/lib/wayfarer/locals.rb +0 -40
  333. data/lib/wayfarer/parsers/json_parser.rb +0 -20
  334. data/lib/wayfarer/parsers/xml_parser.rb +0 -27
  335. data/lib/wayfarer/processor.rb +0 -103
  336. data/lib/wayfarer/routing/custom_rule.rb +0 -21
  337. data/lib/wayfarer/routing/filetypes_rule.rb +0 -20
  338. data/lib/wayfarer/routing/host_rule.rb +0 -19
  339. data/lib/wayfarer/routing/path_rule.rb +0 -54
  340. data/lib/wayfarer/routing/protocol_rule.rb +0 -21
  341. data/lib/wayfarer/routing/query_rule.rb +0 -59
  342. data/lib/wayfarer/routing/rule.rb +0 -114
  343. data/lib/wayfarer/routing/uri_rule.rb +0 -21
  344. data/spec/configuration_spec.rb +0 -26
  345. data/spec/crawl_spec.rb +0 -48
  346. data/spec/finders_spec.rb +0 -49
  347. data/spec/frontiers/memory_bloomfilter_spec.rb +0 -6
  348. data/spec/frontiers/memory_frontier_spec.rb +0 -6
  349. data/spec/frontiers/memory_trie_frontier_spec.rb +0 -6
  350. data/spec/frontiers/normalize_uris_spec.rb +0 -59
  351. data/spec/frontiers/redis_bloomfilter_spec.rb +0 -6
  352. data/spec/frontiers/redis_frontier_spec.rb +0 -6
  353. data/spec/http_adapters/adapter_pool_spec.rb +0 -33
  354. data/spec/http_adapters/net_http_adapter_spec.rb +0 -83
  355. data/spec/http_adapters/selenium_adapter_spec.rb +0 -53
  356. data/spec/integration/callbacks_spec.rb +0 -42
  357. data/spec/integration/locals_spec.rb +0 -106
  358. data/spec/integration/peeking_spec.rb +0 -61
  359. data/spec/job_spec.rb +0 -122
  360. data/spec/processor_spec.rb +0 -31
  361. data/spec/routing/custom_rule_spec.rb +0 -26
  362. data/spec/routing/host_rule_spec.rb +0 -48
  363. data/spec/routing/path_rule_spec.rb +0 -66
  364. data/spec/routing/protocol_rule_spec.rb +0 -26
  365. data/spec/routing/query_rule_spec.rb +0 -124
  366. data/spec/routing/rule_spec.rb +0 -251
  367. data/spec/routing/uri_rule_spec.rb +0 -24
  368. data/spec/shared/frontier.rb +0 -96
  369. data/wayfarer-jruby.gemspec +0 -49
@@ -1,184 +1,39 @@
1
- ---
2
- layout: default
3
- title: Configuration
4
- ---
5
-
6
1
  # Configuration
7
2
 
8
- All job classes base their configuration off the global one.
9
-
10
- {% highlight ruby %}
11
- # Setting a key globally applies to all jobs ...
12
- Wayfarer.config.key = :value
13
-
14
- class DummyJob < Wayfarer::Job
15
- # ... unless a job overrides it
16
- config.key = :other_value
17
- end
18
-
19
- class DummyJob < Wayfarer::Job
20
- # Have it yielded
21
- config { |c| c.key = :other_value }
22
- end
23
- {% endhighlight %}
24
-
25
- ## Recognized keys and values
26
-
27
- ### `print_stacktraces`
28
- * Default: `true`
29
- * Recognized values: Booleans
30
-
31
- Whether to print stacktraces when encounterting unhandled exceptions in job actions. See [Error handling]({{base}}/guides/error_handling.html).
32
-
33
- ---
34
-
35
- ### `reraise_exceptions`
36
-
37
- * Default: `false`
38
- * Recognized values: Booleans
39
-
40
- Whether to crash when encountering unhandled exceptions in job actions. See [Error handling]({{base}}/guides/error_handling.html).
41
-
42
- ---
43
-
44
- ### `allow_circulation`
45
-
46
- * Default: `false`
47
- * Recognized values: Booleans
48
-
49
- Whether URIs may be visited twice.
50
-
51
- <aside class="note">
52
- Allowing circulation might cause your jobs to not terminate.
53
- </aside>
54
-
55
- ---
56
-
57
- ### `normalize_uris`
58
-
59
- * Default: `true`
60
- * Recognized values: Booleans
61
-
62
- Whether to strip fragments, reorder query keys, etc. when staging and caching URIs. Customizable with the `:normalize_uri_options` key. See [normalize_url](https://github.com/rwz/normalize_url).
63
-
64
- ---
65
-
66
- ### `normalize_uri_options`
67
-
68
- * Default: `{}`
69
- * Recognized values: See [normalize_url](https://github.com/rwz/normalize_url).
70
-
71
- ---
72
-
73
- ### `frontier`
74
- * Default: `:memory`
75
- * Recognized values: See [(Redis) frontiers](frontiers.html).
76
-
77
- Which frontier to use.
78
-
79
- <aside class="note">
80
- Bloom filters may yield false positives. See the <a href="https://en.wikipedia.org/wiki/Bloom_filter">Wikipedia article</a>.
81
- </aside>
3
+ Wayfarer can be configured in two ways:
82
4
 
83
- ---
5
+ 1. Using [environment variables](/reference/environment_variables)
6
+ 2. Using runtime configuration
84
7
 
85
- ### `connection_count`
8
+ ## Runtime configuration
86
9
 
87
- * Default: `4`
88
- * Recognized values: Integers
10
+ Wayfarer parses environment variables into a runtime configuration
11
+ `Wayfarer::Config`. The configuration can then be altered or replaced via
12
+ `Wayfarer.config`:
89
13
 
90
- How many threads and HTTP adapters to use (1:1 correspondence).
14
+ ```ruby
15
+ # Which user agent to use to process tasks
16
+ Wayfarer.config.network.agent = :http # or :ferrum, :selenium
91
17
 
92
- ---
18
+ # How many user agents to instantiate
19
+ Wayfarer.config.network.pool_size = 3
93
20
 
94
- ### `http_adapter`
21
+ # How long an agent may be used while processing a task
22
+ Wayfarer.config.network.pool_timeout = 5000
95
23
 
96
- * Default: `:net_http`
97
- * Recognized values: `:net_http`, `:selenium`
24
+ # Ferrum options
25
+ Wayfarer.config.ferrum.options = {}
98
26
 
99
- Which HTTP adapter to use. See [Selenium & Capybara](selenium_capybara.html).
27
+ # Selenium driver to use
28
+ Wayfarer.config.selenium.driver = :chrome
100
29
 
101
- ---
30
+ # Selenium HTTP client read timeout
31
+ Wayfarer.config.selenium.client_timeout = 10 # seconds
102
32
 
103
- ### `connection_timeout`
33
+ # Selenium options
34
+ Wayfarer.config.selenium.options = { url: "http://chrome" }
104
35
 
105
- * Default: `Float::INFINITY`
106
- * Recognized values: Floats
107
-
108
- Time in seconds that a job instance may hold an HTTP adapter. Instances that exceed this time limit raise an exception.
109
-
110
- ---
111
-
112
- ### `max_http_redirects`
113
-
114
- * Default: `3`
115
- * Recognized values: Integers
116
-
117
- How many 3xx redirects to follow.
118
-
119
- <aside class="note">
120
- Has no effect when using the <code>:selenium</code> HTTP adapter.
121
- </aside>
122
-
123
- ---
124
-
125
- ### `selenium_argv`
126
-
127
- * Default: `[:firefox]`
128
- * Recognized values: See [Selenium & Capybara](selenium_capybara.html)
129
-
130
- Argument vector passed to [`Selenium::WebDriver::Driver::for`](http://www.rubydoc.info/gems/selenium-webdriver/Selenium/WebDriver/Driver#for-class_method).
131
-
132
- ---
133
-
134
- ### `redis_opts`
135
-
136
- * Default: `{ host: "localhost", port: 6379 }`
137
- * Recognized values: [See documentation](http://www.rubydoc.info/github/redis/redis-rb/Redis%3Ainitialize)
138
-
139
- Options passed to [`Redis#initialize`](http://www.rubydoc.info/github/redis/redis-rb/Redis%3Ainitialize).
140
-
141
- ---
142
-
143
- ### `bloomfilter_opts`
144
-
145
- * Default:
146
- ```
147
- {
148
- size: 100,
149
- hashes: 2,
150
- seed: 1,
151
- bucket: 3,
152
- raise: false
153
- }
36
+ # HTTP request headers (Selenium is unsupported)
37
+ Wayfarer.config.network.http_headers = { "Field" => "Value" }
154
38
  ```
155
- * Recognized values:
156
- * `size`: Integers; number of buckets in a bloom filter
157
- * `hashes`: Integers; number of hash functions
158
- * `seed`: Integers; seed of hash functions
159
- * `bucket`: Integers; number of bits in a bloom filter bucket
160
- * `raise`: Booleans; whether to raise on bucket overflow
161
-
162
- Options for [bloomfilter-rb](https://github.com/igrigorik/bloomfilter-rb).
163
-
164
- ---
165
-
166
- ### `window_size`
167
-
168
- * Default: `[1024, 768]`
169
- * Recognized values: `[Integer, Integer]`
170
-
171
- Dimensions of browser windows.
172
-
173
- <aside class="note">
174
- Only has an effect when using the <code>:selenium</code> HTTP adapter.
175
- </aside>
176
-
177
- ---
178
-
179
- ### `mustermann_type`
180
-
181
- * Default: `:sinatra`
182
- * Recognized values: [See documentation](https://github.com/sinatra/mustermann)
183
39
 
184
- Which [Mustermann](https://github.com/sinatra/mustermann) pattern type to use.
@@ -0,0 +1,17 @@
1
+ # Debugging
2
+
3
+ [Wayfarer's CLI](/reference/cli/) has two sub-commands that come in handy when
4
+ diagnosing problems in the development workflow.
5
+
6
+ ## Routing a URL from the shell
7
+
8
+ ## `wayfarer route`
9
+
10
+ ### `wayfarer route result JOB URL`
11
+
12
+ : Prints the result of invoking `JOB`'s router with `URL`.
13
+
14
+ ### `wayfarer route tree JOB URL`
15
+
16
+ : Visualises the routing tree result of invoking `JOB`'s router with `URL`.
17
+
@@ -1,46 +1,31 @@
1
- ---
2
- layout: default
3
- title: Error handling
4
- ---
5
-
6
1
  # Error handling
7
- By default, all exceptions raised within actions are swallowed and only their stacktraces printed to stderr. This behaviour can be changed with two configuration keys (see [Configuration]()):
8
-
9
- 1. `print_stacktraces`: Whether to print stacktraces (default: `true`)
10
- 2. `reraise_exceptions`: Whether to crash when encountering unhandled exceptions (default: `false`)
11
-
12
- Here’s an example to illustrate the default behaviour:
13
-
14
- {% highlight ruby %}
15
- class DummyJob < Wayfarer::Job
16
- def example
17
- # Makes this instance fail, but processing goes on
18
- # Prints the stacktrace to stderr
19
- fail "It's okay, life goes on"
20
- end
21
- end
22
- {% endhighlight %}
23
-
24
- The following reraises all exceptions, stops processing and returns with a non-zero exit code:
25
-
26
- {% highlight ruby %}
27
- class DummyJob < Wayfarer::Job
28
- config.reraise_exceptions = true
29
-
30
- def example
31
- fail "This makes the exception bubble up"
32
- end
33
- end
34
- {% endhighlight %}
35
-
36
- And if you don’t want to be bothered with exceptions at all:
37
-
38
- {% highlight ruby %}
39
- class DummyJob < Wayfarer::Job
40
- config.print_stacktraces = false
41
-
42
- def example
43
- fail "No one will know about this ..."
44
- end
45
- end
46
- {% endhighlight %}
2
+
3
+ ## Wayfarer never swallows exceptions
4
+
5
+ * Wayfarer never swallows exceptions.
6
+ * Jobs with unhandled exceptions are not retried.
7
+
8
+ ## Retrying and discarding
9
+
10
+ Wayfarer relies on [Active Job's two error handling facilities](https://guides.rubyonrails.org/active_job_basics.html#exceptions).
11
+
12
+ * `retry_on` to retry jobs a number of times on certain errors:
13
+
14
+ ```ruby
15
+ class DummyJob < Wayfarer::Base
16
+ retry_on MyError, attempts: 3 do |job, error|
17
+ # This block runs once all 3 attempts have failed
18
+ # (1 initial attempt + 2 retries)
19
+ end
20
+ end
21
+ ```
22
+
23
+ * `discard_on` to throw away jobs on certain errors:
24
+
25
+ ```ruby
26
+ class DummyJob < Wayfarer::Base
27
+ discard_on MyError do |job, error|
28
+ # This block runs once and buries the job
29
+ end
30
+ end
31
+ ```
@@ -0,0 +1,101 @@
1
+ # Jobs
2
+
3
+ Jobs are Ruby classes that process [tasks](/guides/tasks) and look as follows:
4
+
5
+ ```ruby
6
+ class DummyJob < Wayfarer::Base
7
+ route { to :index }
8
+
9
+ def index
10
+ end
11
+ end
12
+ ```
13
+
14
+ Here is how to enqueue a task for a URL:
15
+
16
+ ```ruby
17
+ DummyJob.crawl_later("https://example.com")
18
+ ```
19
+
20
+ This is the same as calling the Active Job API directly and passing a task
21
+ and a random batch:
22
+
23
+ ```ruby
24
+ task = Wayfarer::Task.new("https://example.com", SecureRandom.uuid)
25
+ DummyJob.perform_later(task)
26
+ ```
27
+
28
+ A batch can be specified with `::crawl_later`, too:
29
+
30
+ ```ruby
31
+ DummyJob.crawl_later("https://example.com", batch: "my-batch")
32
+ ```
33
+
34
+ ## Current task
35
+
36
+ Jobs consume [tasks](../tasks) from a message queue. The currently processed
37
+ task is accessible like so:
38
+
39
+ ```ruby
40
+ class DummyJob < Wayfarer::Base
41
+ route { to :index }
42
+
43
+ def index
44
+ task.url # => "https://example.com"
45
+ task.batch # => "my-batch"
46
+ end
47
+ end
48
+ ```
49
+
50
+ ## Current page
51
+
52
+ A task's URL contents get fetched into a [page](../pages) object if the task URL
53
+ matched a route:
54
+
55
+ ```ruby
56
+ class DummyJob < Wayfarer::Base
57
+ route { to :index, host: "example.com" }
58
+
59
+ def index
60
+ page.url # => "https://example.com"
61
+ page.body # => "<html>..."
62
+ page.status_code # => 200
63
+ page.headers # { "Content-Type" => ... }
64
+ end
65
+ end
66
+ ```
67
+
68
+ ## URL parameters
69
+
70
+ Jobs can extract data from URLs with their router:
71
+
72
+ ```ruby
73
+ class DummyJob < Wayfarer::Base
74
+ route do
75
+ path "/users/:id/profile"
76
+ end
77
+
78
+ def index
79
+ params[:id] # => "42"
80
+ end
81
+ end
82
+
83
+ DummyJob.crawl_later("https://example.com/users/42/profile")
84
+ ```
85
+
86
+
87
+ ## User agent
88
+
89
+ The HTTP client or automated browser that fetched the URL is available:
90
+
91
+ ```ruby
92
+ Wayfarer.config.network.agent = :ferrum # Chrome DevTools Protocol
93
+
94
+ class DummyJob < Wayfarer::Base
95
+ route { to :index }
96
+
97
+ def index
98
+ browser.save_screenshot("capture.png")
99
+ end
100
+ end
101
+ ```
@@ -0,0 +1,73 @@
1
+ # Navigation
2
+
3
+ Wayfarer has two mechanisms for navigating crawls:
4
+
5
+ * Jobs have a router that decides if a task's URL gets fetched and processed.
6
+ * Jobs can add URLs to a processing set with `#stage`.
7
+
8
+ ## Staging URLs
9
+
10
+ Jobs can turn URLs into tasks within their own batch with `#stage`. Staging a
11
+ URL does not enqueue it immediately. Instead, the URL is added to a processing
12
+ set first.
13
+
14
+ ```ruby
15
+ class DummyJob < Wayfarer::Base
16
+ route { to :index }
17
+
18
+ def index
19
+ stage page.meta.links.all
20
+ end
21
+ end
22
+ ```
23
+
24
+ Once the `index` action method returns, all URLs in `page.meta.links.all`
25
+ are (1) normalized to a canonical form and (2) checked for inclusion in
26
+ the batch's processed URL Redis set. All unprocessed URLs are enqueued as
27
+ tasks within the same batch.
28
+
29
+ `#stage` can be called arbitrarily often, with invalid URLs too, as they are
30
+ filtered out behind the scenes:
31
+
32
+ ```ruby
33
+ def index
34
+ stage "_bro:ken@url/" # => ["_bro:ken@url/"]
35
+ end
36
+ ```
37
+
38
+ See also: [Performance: Stage less URLs](/guides/performance)
39
+
40
+ !!! attention "Failing action methods do not enqueue tasks"
41
+
42
+ If an action method fails as in:
43
+
44
+ ```ruby
45
+ def index
46
+ stage page.meta.links.all
47
+ fail "Error occured"
48
+ end
49
+ ```
50
+
51
+ None of the staged URLs are enqueued as tasks. Jobs that raise an exception
52
+ should get retried, or the exception should be handled.
53
+
54
+
55
+ ## Routing URLs
56
+
57
+ In the following example, the task is written to the message queue, but the
58
+ job's routes do not match the URL. When the task gets consumed, the URL does not
59
+ get fetched and the action method not called.
60
+
61
+ ```ruby
62
+ class DummyJob < Wayfarer::Base
63
+ route do
64
+ host "example.com", path: "/users/:user_id", to: :user
65
+ end
66
+
67
+ # ...
68
+ end
69
+
70
+ DummyJob.crawl_later("https://mismatching.host/users/42")
71
+ ```
72
+
73
+
@@ -0,0 +1,94 @@
1
+ # Networking
2
+
3
+ Wayfarer navigates the web in two ways:
4
+
5
+ 1. Via plain HTTP requests
6
+ 2. By automating browsers
7
+
8
+ Both options are mutually exclusive per Ruby process.
9
+
10
+ ## User agents
11
+
12
+ A user agent is an entity that knows how to retrieve the contents behind a URL.
13
+
14
+ The user agent can be configured via the global configuration:
15
+
16
+ ```ruby
17
+ Wayfarer.config.network.agent = :http # or :ferrum, :selenium
18
+ ```
19
+
20
+ ## Connection pooling
21
+
22
+ Wayfarer keeps user agents within a connection pool. When a job executes
23
+ and needs to retrieve the contents behind a URL, an agent is checked out from
24
+ the pool.
25
+
26
+ The pool has a constant size and it should equal the number of threads the
27
+ underlying message queue operates with. The size can be configured via the
28
+ global configuration:
29
+
30
+ ```ruby
31
+ Wayfarer.config.network.pool_size = 8
32
+ ```
33
+
34
+ ### Timeouts
35
+
36
+ user agents may stay checked out from the pool by jobs for a limited time
37
+ only. Once this time limit is exceeded, a `ConnectionPool::TimeoutError`
38
+ exception is raised. This places a hard time limit on every job.
39
+
40
+ The timeout can be configured via the global configuration:
41
+
42
+ ```ruby
43
+ Wayfarer.config.network.pool_timeout = 20 # seconds
44
+ ```
45
+
46
+ Because jobs with unhandled exceptions fail, explicit error handling is required
47
+ if retries are desired:
48
+
49
+ ```ruby
50
+ class DummyJob < Wayfarer::Base
51
+ retry_on ConnectionPool::TimeoutError, attempts: 3
52
+ end
53
+ ```
54
+
55
+ ## Agent-specific client timeouts
56
+
57
+ The time in seconds it may take to communicate with remote browser processes can
58
+ be configured globally per agent:
59
+
60
+ ```ruby
61
+ Wayfarer.config.ferrum.options = { timeout: 5 }
62
+ Wayfarer.config.selenium.client_timeout = 60
63
+ ```
64
+
65
+ ### Shared state
66
+
67
+ As user agents get checked in and out continously between jobs, their state
68
+ carries over from job to job, too.
69
+
70
+ For browser automation, this means:
71
+
72
+ * A job finds the browser at the last URL the previous job has left off.
73
+ * The browser's cookies might have been set, or other client-side state might
74
+ exist that significantly affects a page's behaviour.
75
+
76
+ ## HTTP redirect handling
77
+
78
+ Browsers follow redirects transparently when they are navigated to a URL.
79
+
80
+ When using plain HTTP, redirect URLs are enqueued transparently within the same
81
+ batch. URLs that result in 3xx responses will not be retrieved again within
82
+ their batch.
83
+
84
+ ## HTTP request headers
85
+
86
+ Request headers can be configured via the global configuration:
87
+
88
+ ```ruby
89
+ Wayfarer.config.network.http_headers = { "Field" => "Value" }
90
+ ```
91
+
92
+ !!! attention "Partial support"
93
+
94
+ Selenium does not support configuring HTTP request headers.
@@ -0,0 +1,52 @@
1
+ # Pages
2
+
3
+ Retrieved pages take the shape of `Wayfarer::Page` objects and are available
4
+ to jobs:
5
+
6
+ ```ruby
7
+ class DummyJob < Wayfarer::Worker
8
+ route { to :index }
9
+
10
+ def index
11
+ page # => #<Wayfarer::Page ...>
12
+
13
+ page.url # => "https://example.com"
14
+ page.body # => "<html>..."
15
+ page.status_code # => 200
16
+ page.headers # => { "Content-Type" => ... }
17
+
18
+ # A MetaInspector object for accessing page meta data.
19
+ # See: https://github.com/metainspector/metainspector
20
+ # Examples:
21
+ page.meta.links.internal
22
+ page.meta.images.favicon
23
+ page.meta.description
24
+ page.meta.feeds
25
+ end
26
+ end
27
+ ```
28
+
29
+ ## Live pages
30
+
31
+ When automating browsers, it is possible the page changes significantly at
32
+ runtime, for example due to JavaScript altering the DOM or URL.
33
+
34
+ To access a page reflecting the current browser state, pass the `live` keyword:
35
+
36
+ ```ruby
37
+ class DummyJob < Wayfarer::Worker
38
+ route { to :index }
39
+
40
+ def index
41
+ page # => #<Wayfarer::Page ...>
42
+
43
+ # Replaces the current Page object with a newer one,
44
+ # taking into account the DOM as currently rendered by the browser.
45
+ # Effectful only when automating browsers, no-op when using plain
46
+ # HTTP.
47
+ page(live: true)
48
+
49
+ page # => The live page returned above
50
+ end
51
+ end
52
+ ```