wayfarer 0.0.3 → 0.4.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (369) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/ci.yaml +32 -0
  3. data/.gitignore +3 -4
  4. data/.rubocop.yml +25 -9
  5. data/.ruby-version +1 -1
  6. data/Dockerfile +5 -0
  7. data/Gemfile +1 -7
  8. data/Gemfile.lock +221 -0
  9. data/RELEASING.md +17 -0
  10. data/Rakefile +38 -90
  11. data/bin/wayfarer +1 -111
  12. data/docker-compose.yml +32 -0
  13. data/docs/cookbook/batch_routing.md +22 -0
  14. data/docs/cookbook/consent_screen.md +36 -0
  15. data/docs/cookbook/executing_javascript.md +41 -0
  16. data/docs/cookbook/querying_html.md +42 -0
  17. data/docs/cookbook/screenshots.md +27 -0
  18. data/docs/cookbook/user_agent.md +7 -0
  19. data/docs/guides/browser_automation/capybara.md +69 -0
  20. data/docs/guides/browser_automation/custom_adapters.md +100 -0
  21. data/docs/guides/browser_automation/ferrum.md +39 -0
  22. data/docs/guides/browser_automation/selenium.md +63 -0
  23. data/docs/guides/callbacks.md +131 -31
  24. data/docs/guides/configuration.md +24 -169
  25. data/docs/guides/debugging.md +17 -0
  26. data/docs/guides/error_handling.md +30 -45
  27. data/docs/guides/jobs.md +101 -0
  28. data/docs/guides/navigation.md +73 -0
  29. data/docs/guides/networking.md +94 -0
  30. data/docs/guides/pages.md +52 -0
  31. data/docs/guides/performance.md +130 -0
  32. data/docs/guides/reliability.md +41 -0
  33. data/docs/guides/routing/steering.md +30 -0
  34. data/docs/guides/tasks.md +14 -0
  35. data/docs/index.md +40 -66
  36. data/docs/reference/api/base.md +48 -0
  37. data/docs/reference/api/route.md +182 -0
  38. data/docs/reference/cli.md +61 -0
  39. data/docs/reference/configuration_keys.md +42 -0
  40. data/docs/reference/environment_variables.md +83 -0
  41. data/lib/wayfarer/base.rb +50 -0
  42. data/lib/wayfarer/callbacks.rb +71 -0
  43. data/lib/wayfarer/cli/base.rb +27 -0
  44. data/lib/wayfarer/cli/generate.rb +17 -0
  45. data/lib/wayfarer/cli/job.rb +60 -0
  46. data/lib/wayfarer/cli/route.rb +29 -0
  47. data/lib/wayfarer/cli/route_printer.rb +116 -0
  48. data/lib/wayfarer/cli/runner.rb +34 -0
  49. data/lib/wayfarer/cli/templates/Gemfile.tt +5 -0
  50. data/lib/wayfarer/cli/templates/job.rb.tt +10 -0
  51. data/lib/wayfarer/config/capybara.rb +10 -0
  52. data/lib/wayfarer/config/ferrum.rb +11 -0
  53. data/lib/wayfarer/config/networking.rb +26 -0
  54. data/lib/wayfarer/config/redis.rb +14 -0
  55. data/lib/wayfarer/config/root.rb +11 -0
  56. data/lib/wayfarer/config/selenium.rb +21 -0
  57. data/lib/wayfarer/config/strconv.rb +45 -0
  58. data/lib/wayfarer/config/struct.rb +72 -0
  59. data/lib/wayfarer/gc.rb +15 -0
  60. data/lib/wayfarer/middleware/chain.rb +19 -0
  61. data/lib/wayfarer/middleware/dedup.rb +25 -0
  62. data/lib/wayfarer/middleware/fetch.rb +47 -0
  63. data/lib/wayfarer/middleware/normalize.rb +25 -0
  64. data/lib/wayfarer/middleware/router.rb +53 -0
  65. data/lib/wayfarer/middleware/stage.rb +23 -0
  66. data/lib/wayfarer/middleware/worker.rb +30 -0
  67. data/lib/wayfarer/networking/capybara.rb +28 -0
  68. data/lib/wayfarer/networking/context.rb +36 -0
  69. data/lib/wayfarer/networking/ferrum.rb +35 -0
  70. data/lib/wayfarer/networking/http.rb +34 -0
  71. data/lib/wayfarer/networking/pool.rb +40 -0
  72. data/lib/wayfarer/networking/result.rb +18 -0
  73. data/lib/wayfarer/networking/selenium.rb +43 -0
  74. data/lib/wayfarer/networking/strategy.rb +38 -0
  75. data/lib/wayfarer/page.rb +17 -74
  76. data/lib/wayfarer/parsing/json.rb +17 -0
  77. data/lib/wayfarer/parsing/xml.rb +17 -0
  78. data/lib/wayfarer/redis/.#barrier.rb +1 -0
  79. data/lib/wayfarer/redis/barrier.rb +36 -0
  80. data/lib/wayfarer/redis/connection.rb +13 -0
  81. data/lib/wayfarer/redis/counter.rb +29 -0
  82. data/lib/wayfarer/redis/pool.rb +20 -0
  83. data/lib/wayfarer/redis/version.rb +19 -0
  84. data/lib/wayfarer/routing/dsl.rb +57 -0
  85. data/lib/wayfarer/routing/matchers/custom.rb +25 -0
  86. data/lib/wayfarer/routing/matchers/host.rb +19 -0
  87. data/lib/wayfarer/routing/matchers/path.rb +49 -0
  88. data/lib/wayfarer/routing/matchers/query.rb +63 -0
  89. data/lib/wayfarer/routing/matchers/scheme.rb +17 -0
  90. data/lib/wayfarer/routing/matchers/suffix.rb +17 -0
  91. data/lib/wayfarer/routing/matchers/url.rb +17 -0
  92. data/lib/wayfarer/routing/path_finder.rb +46 -0
  93. data/lib/wayfarer/routing/result.rb +15 -0
  94. data/lib/wayfarer/routing/root_route.rb +7 -0
  95. data/lib/wayfarer/routing/route.rb +47 -0
  96. data/lib/wayfarer/routing/router.rb +10 -54
  97. data/lib/wayfarer/routing/target_route.rb +7 -0
  98. data/lib/wayfarer/serializer.rb +17 -0
  99. data/lib/wayfarer/stringify.rb +47 -0
  100. data/lib/wayfarer/task.rb +34 -0
  101. data/lib/wayfarer.rb +48 -57
  102. data/mkdocs.yml +47 -0
  103. data/requirements.txt +1 -0
  104. data/spec/base_spec.rb +233 -0
  105. data/spec/callbacks_spec.rb +102 -0
  106. data/spec/cli/generate_spec.rb +39 -0
  107. data/spec/cli/job_spec.rb +74 -0
  108. data/spec/cli/version_spec.rb +13 -0
  109. data/spec/config/capybara_spec.rb +18 -0
  110. data/spec/config/ferrum_spec.rb +24 -0
  111. data/spec/config/networking_spec.rb +73 -0
  112. data/spec/config/redis_spec.rb +32 -0
  113. data/spec/config/root_spec.rb +31 -0
  114. data/spec/config/selenium_spec.rb +56 -0
  115. data/spec/config/strconv_spec.rb +58 -0
  116. data/spec/config/struct_spec.rb +66 -0
  117. data/spec/factories/middleware.rb +15 -0
  118. data/spec/factories/page.rb +78 -0
  119. data/spec/factories/task.rb +12 -0
  120. data/spec/fixtures/dummy_job.rb +7 -0
  121. data/spec/gc_spec.rb +63 -0
  122. data/spec/middleware/chain_spec.rb +96 -0
  123. data/spec/middleware/dedup_spec.rb +76 -0
  124. data/spec/middleware/fetch_spec.rb +100 -0
  125. data/spec/middleware/normalize_spec.rb +28 -0
  126. data/spec/middleware/router_spec.rb +80 -0
  127. data/spec/middleware/stage_spec.rb +39 -0
  128. data/spec/middleware/worker_spec.rb +117 -0
  129. data/spec/networking/capybara_spec.rb +12 -0
  130. data/spec/networking/context_spec.rb +127 -0
  131. data/spec/networking/ferrum_spec.rb +12 -0
  132. data/spec/networking/http_spec.rb +12 -0
  133. data/spec/networking/pool_spec.rb +67 -0
  134. data/spec/networking/selenium_spec.rb +12 -0
  135. data/spec/networking/strategy.rb +170 -0
  136. data/spec/page_spec.rb +21 -12
  137. data/spec/{parsers/json_parser_spec.rb → parsing/json_spec.rb} +5 -4
  138. data/spec/{parsers/xml_parser_spec.rb → parsing/xml_spec.rb} +3 -2
  139. data/spec/redis/barrier_spec.rb +78 -0
  140. data/spec/redis/counter_spec.rb +32 -0
  141. data/spec/redis/pool_spec.rb +18 -0
  142. data/spec/redis/version_spec.rb +13 -0
  143. data/spec/routing/dsl_spec.rb +98 -0
  144. data/spec/routing/integration_spec.rb +110 -0
  145. data/spec/routing/matchers/custom_spec.rb +31 -0
  146. data/spec/routing/matchers/host_spec.rb +49 -0
  147. data/spec/routing/matchers/path_spec.rb +43 -0
  148. data/spec/routing/matchers/query_spec.rb +137 -0
  149. data/spec/routing/matchers/scheme_spec.rb +25 -0
  150. data/spec/routing/{filetypes_rule_spec.rb → matchers/suffix_spec.rb} +14 -13
  151. data/spec/routing/matchers/uri_spec.rb +27 -0
  152. data/spec/routing/path_finder_spec.rb +33 -0
  153. data/spec/routing/root_route_spec.rb +29 -0
  154. data/spec/routing/route_spec.rb +43 -0
  155. data/spec/routing/router_spec.rb +13 -56
  156. data/spec/spec_helpers.rb +73 -38
  157. data/spec/stringify_spec.rb +23 -0
  158. data/{support → spec/support}/static/finders.html +0 -0
  159. data/{support → spec/support}/static/graph/details/a.html +0 -0
  160. data/{support → spec/support}/static/graph/details/b.html +0 -0
  161. data/{support → spec/support}/static/graph/index.html +0 -0
  162. data/{support → spec/support}/static/json/dummy.json +0 -0
  163. data/{support → spec/support}/static/links/links.html +0 -0
  164. data/{support → spec/support}/static/xml/dummy.xml +0 -0
  165. data/{support → spec/support}/test_app.rb +9 -2
  166. data/spec/task_spec.rb +27 -0
  167. data/spec/wayfarer_spec.rb +2 -13
  168. data/wayfarer.gemspec +40 -42
  169. metadata +234 -361
  170. data/.travis.yml +0 -5
  171. data/Changelog.md +0 -10
  172. data/README.md +0 -21
  173. data/benchmark/frontiers.rb +0 -143
  174. data/docs/.gitignore +0 -2
  175. data/docs/_config.yml +0 -15
  176. data/docs/_includes/base.html +0 -7
  177. data/docs/_includes/head.html +0 -10
  178. data/docs/_includes/navigation.html +0 -187
  179. data/docs/_layouts/default.html +0 -42
  180. data/docs/_sass/base.scss +0 -439
  181. data/docs/_sass/variables.scss +0 -24
  182. data/docs/_sass/vendor/bourbon/_bourbon-deprecate.scss +0 -19
  183. data/docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss +0 -425
  184. data/docs/_sass/vendor/bourbon/_bourbon.scss +0 -90
  185. data/docs/_sass/vendor/bourbon/addons/_border-color.scss +0 -29
  186. data/docs/_sass/vendor/bourbon/addons/_border-radius.scss +0 -48
  187. data/docs/_sass/vendor/bourbon/addons/_border-style.scss +0 -28
  188. data/docs/_sass/vendor/bourbon/addons/_border-width.scss +0 -28
  189. data/docs/_sass/vendor/bourbon/addons/_buttons.scss +0 -69
  190. data/docs/_sass/vendor/bourbon/addons/_clearfix.scss +0 -25
  191. data/docs/_sass/vendor/bourbon/addons/_ellipsis.scss +0 -30
  192. data/docs/_sass/vendor/bourbon/addons/_font-stacks.scss +0 -31
  193. data/docs/_sass/vendor/bourbon/addons/_hide-text.scss +0 -27
  194. data/docs/_sass/vendor/bourbon/addons/_margin.scss +0 -29
  195. data/docs/_sass/vendor/bourbon/addons/_padding.scss +0 -29
  196. data/docs/_sass/vendor/bourbon/addons/_position.scss +0 -51
  197. data/docs/_sass/vendor/bourbon/addons/_prefixer.scss +0 -66
  198. data/docs/_sass/vendor/bourbon/addons/_retina-image.scss +0 -27
  199. data/docs/_sass/vendor/bourbon/addons/_size.scss +0 -56
  200. data/docs/_sass/vendor/bourbon/addons/_text-inputs.scss +0 -118
  201. data/docs/_sass/vendor/bourbon/addons/_timing-functions.scss +0 -34
  202. data/docs/_sass/vendor/bourbon/addons/_triangle.scss +0 -63
  203. data/docs/_sass/vendor/bourbon/addons/_word-wrap.scss +0 -29
  204. data/docs/_sass/vendor/bourbon/css3/_animation.scss +0 -61
  205. data/docs/_sass/vendor/bourbon/css3/_appearance.scss +0 -5
  206. data/docs/_sass/vendor/bourbon/css3/_backface-visibility.scss +0 -5
  207. data/docs/_sass/vendor/bourbon/css3/_background-image.scss +0 -44
  208. data/docs/_sass/vendor/bourbon/css3/_background.scss +0 -57
  209. data/docs/_sass/vendor/bourbon/css3/_border-image.scss +0 -61
  210. data/docs/_sass/vendor/bourbon/css3/_calc.scss +0 -6
  211. data/docs/_sass/vendor/bourbon/css3/_columns.scss +0 -67
  212. data/docs/_sass/vendor/bourbon/css3/_filter.scss +0 -6
  213. data/docs/_sass/vendor/bourbon/css3/_flex-box.scss +0 -327
  214. data/docs/_sass/vendor/bourbon/css3/_font-face.scss +0 -29
  215. data/docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss +0 -6
  216. data/docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss +0 -12
  217. data/docs/_sass/vendor/bourbon/css3/_hyphens.scss +0 -6
  218. data/docs/_sass/vendor/bourbon/css3/_image-rendering.scss +0 -15
  219. data/docs/_sass/vendor/bourbon/css3/_keyframes.scss +0 -38
  220. data/docs/_sass/vendor/bourbon/css3/_linear-gradient.scss +0 -40
  221. data/docs/_sass/vendor/bourbon/css3/_perspective.scss +0 -12
  222. data/docs/_sass/vendor/bourbon/css3/_placeholder.scss +0 -10
  223. data/docs/_sass/vendor/bourbon/css3/_radial-gradient.scss +0 -40
  224. data/docs/_sass/vendor/bourbon/css3/_selection.scss +0 -44
  225. data/docs/_sass/vendor/bourbon/css3/_text-decoration.scss +0 -27
  226. data/docs/_sass/vendor/bourbon/css3/_transform.scss +0 -21
  227. data/docs/_sass/vendor/bourbon/css3/_transition.scss +0 -81
  228. data/docs/_sass/vendor/bourbon/css3/_user-select.scss +0 -5
  229. data/docs/_sass/vendor/bourbon/functions/_assign-inputs.scss +0 -16
  230. data/docs/_sass/vendor/bourbon/functions/_contains-falsy.scss +0 -25
  231. data/docs/_sass/vendor/bourbon/functions/_contains.scss +0 -31
  232. data/docs/_sass/vendor/bourbon/functions/_is-length.scss +0 -16
  233. data/docs/_sass/vendor/bourbon/functions/_is-light.scss +0 -26
  234. data/docs/_sass/vendor/bourbon/functions/_is-number.scss +0 -16
  235. data/docs/_sass/vendor/bourbon/functions/_is-size.scss +0 -23
  236. data/docs/_sass/vendor/bourbon/functions/_modular-scale.scss +0 -74
  237. data/docs/_sass/vendor/bourbon/functions/_px-to-em.scss +0 -24
  238. data/docs/_sass/vendor/bourbon/functions/_px-to-rem.scss +0 -26
  239. data/docs/_sass/vendor/bourbon/functions/_shade.scss +0 -24
  240. data/docs/_sass/vendor/bourbon/functions/_strip-units.scss +0 -22
  241. data/docs/_sass/vendor/bourbon/functions/_tint.scss +0 -24
  242. data/docs/_sass/vendor/bourbon/functions/_transition-property-name.scss +0 -37
  243. data/docs/_sass/vendor/bourbon/functions/_unpack.scss +0 -32
  244. data/docs/_sass/vendor/bourbon/helpers/_convert-units.scss +0 -26
  245. data/docs/_sass/vendor/bourbon/helpers/_directional-values.scss +0 -108
  246. data/docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss +0 -53
  247. data/docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss +0 -24
  248. data/docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss +0 -35
  249. data/docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss +0 -51
  250. data/docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss +0 -77
  251. data/docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss +0 -41
  252. data/docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss +0 -74
  253. data/docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss +0 -55
  254. data/docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss +0 -28
  255. data/docs/_sass/vendor/bourbon/helpers/_render-gradients.scss +0 -31
  256. data/docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss +0 -15
  257. data/docs/_sass/vendor/bourbon/helpers/_str-to-num.scss +0 -55
  258. data/docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss +0 -7
  259. data/docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss +0 -8
  260. data/docs/_sass/vendor/bourbon/settings/_prefixer.scss +0 -9
  261. data/docs/_sass/vendor/bourbon/settings/_px-to-em.scss +0 -1
  262. data/docs/_sass/vendor/neat/_neat-helpers.scss +0 -11
  263. data/docs/_sass/vendor/neat/_neat.scss +0 -23
  264. data/docs/_sass/vendor/neat/functions/_new-breakpoint.scss +0 -49
  265. data/docs/_sass/vendor/neat/functions/_private.scss +0 -114
  266. data/docs/_sass/vendor/neat/grid/_box-sizing.scss +0 -15
  267. data/docs/_sass/vendor/neat/grid/_direction-context.scss +0 -33
  268. data/docs/_sass/vendor/neat/grid/_display-context.scss +0 -28
  269. data/docs/_sass/vendor/neat/grid/_fill-parent.scss +0 -22
  270. data/docs/_sass/vendor/neat/grid/_media.scss +0 -92
  271. data/docs/_sass/vendor/neat/grid/_omega.scss +0 -87
  272. data/docs/_sass/vendor/neat/grid/_outer-container.scss +0 -34
  273. data/docs/_sass/vendor/neat/grid/_pad.scss +0 -25
  274. data/docs/_sass/vendor/neat/grid/_private.scss +0 -35
  275. data/docs/_sass/vendor/neat/grid/_row.scss +0 -52
  276. data/docs/_sass/vendor/neat/grid/_shift.scss +0 -50
  277. data/docs/_sass/vendor/neat/grid/_span-columns.scss +0 -94
  278. data/docs/_sass/vendor/neat/grid/_to-deprecate.scss +0 -97
  279. data/docs/_sass/vendor/neat/grid/_visual-grid.scss +0 -42
  280. data/docs/_sass/vendor/neat/mixins/_clearfix.scss +0 -25
  281. data/docs/_sass/vendor/neat/settings/_disable-warnings.scss +0 -13
  282. data/docs/_sass/vendor/neat/settings/_grid.scss +0 -51
  283. data/docs/_sass/vendor/neat/settings/_visual-grid.scss +0 -27
  284. data/docs/_sass/vendor/normalize-3.0.2.scss +0 -427
  285. data/docs/_sass/vendor/pygments.scss +0 -356
  286. data/docs/automating_browsers/capybara.md +0 -70
  287. data/docs/css/screen.scss +0 -7
  288. data/docs/guides/cli.md +0 -52
  289. data/docs/guides/frontiers.md +0 -93
  290. data/docs/guides/halting.md +0 -23
  291. data/docs/guides/job_queues.md +0 -26
  292. data/docs/guides/locals.md +0 -36
  293. data/docs/guides/logging.md +0 -22
  294. data/docs/guides/page_objects.md +0 -67
  295. data/docs/guides/peeking.md +0 -46
  296. data/docs/guides/selenium_capybara.md +0 -100
  297. data/docs/guides/tutorial.md +0 -452
  298. data/docs/js/navigation.js +0 -11
  299. data/docs/misc/contributing.md +0 -20
  300. data/docs/misc/testing.md +0 -11
  301. data/docs/recipes/authentication.md +0 -23
  302. data/docs/recipes/csv.md +0 -29
  303. data/docs/recipes/javascript.md +0 -20
  304. data/docs/recipes/multiple_uris.md +0 -18
  305. data/docs/recipes/screenshots.md +0 -20
  306. data/docs/routing/custom_rules.md +0 -16
  307. data/docs/routing/filetypes_rules.md +0 -21
  308. data/docs/routing/host_rules.md +0 -24
  309. data/docs/routing/path_rules.md +0 -33
  310. data/docs/routing/protocol_rules.md +0 -17
  311. data/docs/routing/query_rules.md +0 -69
  312. data/docs/routing/routes.md +0 -96
  313. data/docs/routing/uri_rules.md +0 -18
  314. data/examples/collect_github_issues.rb +0 -65
  315. data/examples/find_foobar_on_wikipedia.rb +0 -23
  316. data/lib/wayfarer/configuration.rb +0 -86
  317. data/lib/wayfarer/crawl.rb +0 -79
  318. data/lib/wayfarer/crawl_observer.rb +0 -103
  319. data/lib/wayfarer/dispatcher.rb +0 -104
  320. data/lib/wayfarer/finders.rb +0 -61
  321. data/lib/wayfarer/frontiers/frontier.rb +0 -79
  322. data/lib/wayfarer/frontiers/memory_bloomfilter.rb +0 -32
  323. data/lib/wayfarer/frontiers/memory_frontier.rb +0 -76
  324. data/lib/wayfarer/frontiers/memory_trie_frontier.rb +0 -39
  325. data/lib/wayfarer/frontiers/normalize_uris.rb +0 -48
  326. data/lib/wayfarer/frontiers/redis_bloomfilter.rb +0 -34
  327. data/lib/wayfarer/frontiers/redis_frontier.rb +0 -83
  328. data/lib/wayfarer/http_adapters/adapter_pool.rb +0 -62
  329. data/lib/wayfarer/http_adapters/net_http_adapter.rb +0 -77
  330. data/lib/wayfarer/http_adapters/selenium_adapter.rb +0 -80
  331. data/lib/wayfarer/job.rb +0 -211
  332. data/lib/wayfarer/locals.rb +0 -40
  333. data/lib/wayfarer/parsers/json_parser.rb +0 -20
  334. data/lib/wayfarer/parsers/xml_parser.rb +0 -27
  335. data/lib/wayfarer/processor.rb +0 -103
  336. data/lib/wayfarer/routing/custom_rule.rb +0 -21
  337. data/lib/wayfarer/routing/filetypes_rule.rb +0 -20
  338. data/lib/wayfarer/routing/host_rule.rb +0 -19
  339. data/lib/wayfarer/routing/path_rule.rb +0 -54
  340. data/lib/wayfarer/routing/protocol_rule.rb +0 -21
  341. data/lib/wayfarer/routing/query_rule.rb +0 -59
  342. data/lib/wayfarer/routing/rule.rb +0 -114
  343. data/lib/wayfarer/routing/uri_rule.rb +0 -21
  344. data/spec/configuration_spec.rb +0 -26
  345. data/spec/crawl_spec.rb +0 -48
  346. data/spec/finders_spec.rb +0 -49
  347. data/spec/frontiers/memory_bloomfilter_spec.rb +0 -6
  348. data/spec/frontiers/memory_frontier_spec.rb +0 -6
  349. data/spec/frontiers/memory_trie_frontier_spec.rb +0 -6
  350. data/spec/frontiers/normalize_uris_spec.rb +0 -59
  351. data/spec/frontiers/redis_bloomfilter_spec.rb +0 -6
  352. data/spec/frontiers/redis_frontier_spec.rb +0 -6
  353. data/spec/http_adapters/adapter_pool_spec.rb +0 -33
  354. data/spec/http_adapters/net_http_adapter_spec.rb +0 -83
  355. data/spec/http_adapters/selenium_adapter_spec.rb +0 -53
  356. data/spec/integration/callbacks_spec.rb +0 -42
  357. data/spec/integration/locals_spec.rb +0 -106
  358. data/spec/integration/peeking_spec.rb +0 -61
  359. data/spec/job_spec.rb +0 -122
  360. data/spec/processor_spec.rb +0 -31
  361. data/spec/routing/custom_rule_spec.rb +0 -26
  362. data/spec/routing/host_rule_spec.rb +0 -48
  363. data/spec/routing/path_rule_spec.rb +0 -66
  364. data/spec/routing/protocol_rule_spec.rb +0 -26
  365. data/spec/routing/query_rule_spec.rb +0 -124
  366. data/spec/routing/rule_spec.rb +0 -251
  367. data/spec/routing/uri_rule_spec.rb +0 -24
  368. data/spec/shared/frontier.rb +0 -96
  369. data/wayfarer-jruby.gemspec +0 -49
@@ -0,0 +1,83 @@
1
+ # Environment Variables
2
+
3
+ ## String formats
4
+
5
+ Environment variable values can be parsed to Hash or Array at runtime
6
+ with the following syntaxes:
7
+
8
+ * Hash: Variable string `a:1,b:2,c:3` parses to `{a:1, b:2, c:3}` at runtime
9
+ * Array: Variable string `a,b,c` parses to `[:a, :b, :c]` at runtime
10
+
11
+ ## Variables
12
+
13
+ ### `WAYFARER_AGENT`
14
+ : Either `ferrum`, `selenium` or `http`.
15
+
16
+ * Type: String
17
+ * Key: `config.agent`
18
+ * Default value: `:http`
19
+
20
+ ### `WAYFARER_POOL_SIZE`
21
+ : Number of user agents to maintain.
22
+
23
+ * Type: Integer
24
+ * Key: `config.pool_size`
25
+ * Default value: `1`
26
+
27
+ ### `WAYFARER_POOL_TIMEOUT`
28
+ : How long a user agent may remain checked out until the owning job
29
+ fails.
30
+
31
+ * Type: Integer
32
+ * Key: `config.agent_pool_timeout`
33
+ * Default value: `1`
34
+
35
+ ---
36
+
37
+ ### `WAYFARER_FERRUM_OPTIONS`
38
+ : Key/value options passed to `Ferrum::Browser.new`.
39
+
40
+ * Type: Hash
41
+ * Key: `config.ferrum_options`
42
+ * Default value: `{}`
43
+
44
+ ---
45
+
46
+ ### `WAYFARER_SELENIUM_DRIVER`
47
+ : Driver passed to `Selenium::WebDriver.for`.
48
+
49
+ * Type: Symbol
50
+ * Key: `config.selenium_driver`
51
+ * Default value: `:chrome`
52
+
53
+ ---
54
+
55
+ ### `WAYFARER_SELENIUM_OPTIONS`
56
+ : Options passed to `Selenium::WebDriver.for`.
57
+
58
+ * Type: Hash
59
+ * Key: `config.selenium_options`
60
+ * Default value: `{}`
61
+
62
+ ---
63
+
64
+ ### `WAYFARER_SELENIUM_CLIENT_TIMEOUT`
65
+ : Selenium HTTP client timeout (seconds).
66
+
67
+ * Type: Integer
68
+ * Key: `config.selenium_client_timeout`
69
+ * Default value: `60`
70
+
71
+ ---
72
+
73
+ ### `WAYFARER_HTTP_HEADERS`
74
+ : HTTP request headers used when retrieving pages.
75
+
76
+ * Type: Hash
77
+ * Key: `config.http_headers`
78
+ * Default value: `{}`
79
+
80
+ !!! attention "Partial support"
81
+
82
+ Selenium does not support configuring HTTP request headers.
83
+
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ class Base < ActiveJob::Base
5
+ include Wayfarer::Middleware::Worker
6
+ extend Forwardable
7
+
8
+ after_enqueue do |_job|
9
+ task.counter.increment
10
+ end
11
+
12
+ after_perform { |job| GC.new(job).run }
13
+
14
+ rescue_from(StandardError) { GC.new(self).run }
15
+
16
+ def self.retry_on(*argv)
17
+ super(*argv) do |job, error|
18
+ job.task.barrier.seen?(job.task.url)
19
+ GC.new(job).run
20
+ yield job, error if block_given?
21
+ end
22
+ end
23
+
24
+ def self.discard_on(*argv)
25
+ super(*argv) do |job, error|
26
+ job.task.barrier.seen?(job.task.url)
27
+ GC.new(job).run
28
+ yield job, error if block_given?
29
+ end
30
+ end
31
+
32
+ def self.crawl_later(url, batch: SecureRandom.uuid)
33
+ perform_later(Task.new(url, batch))
34
+ end
35
+
36
+ def retry_job(...)
37
+ super(...) # increments the counter by re-enqueuing the job
38
+ task.counter.decrement
39
+ end
40
+
41
+ def perform(task)
42
+ task.job = self
43
+ chain.call(task)
44
+ end
45
+
46
+ def task
47
+ arguments.first
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ module Callbacks
5
+ TERMINATOR = ->(_target, result) { result.call == false }
6
+ OPTIONS = { terminator: TERMINATOR, skip_after_callbacks_if_terminated: true }.freeze
7
+
8
+ ConditionalCallback = Struct.new(:job, :filters) do
9
+ def run(method, &block)
10
+ return if only && !applies?(only)
11
+ return if except && applies?(except)
12
+
13
+ return job.send(method) if method
14
+
15
+ job.instance_eval(&block)
16
+ end
17
+
18
+ private
19
+
20
+ def applies?(condition)
21
+ case condition
22
+ when Symbol then condition == action
23
+ when Enumerable then condition&.include?(action)
24
+ end
25
+ end
26
+
27
+ def only
28
+ filters[:only]
29
+ end
30
+
31
+ def except
32
+ filters[:except]
33
+ end
34
+
35
+ def action
36
+ task.metadata.action
37
+ end
38
+
39
+ def task
40
+ job.task
41
+ end
42
+ end
43
+
44
+ def self.included(base)
45
+ base.include(ActiveSupport::Callbacks)
46
+ base.extend(ClassMethods)
47
+
48
+ base.class_eval do
49
+ define_callbacks(:fetch, OPTIONS)
50
+ define_callbacks(:action, OPTIONS)
51
+ define_callbacks(:batch, OPTIONS)
52
+
53
+ define(:fetch, :before)
54
+ define(:action, :before)
55
+ define(:batch, :after)
56
+ end
57
+ end
58
+
59
+ module ClassMethods
60
+ private
61
+
62
+ def define(name, stage)
63
+ define_singleton_method([stage, name].join("_")) do |method = nil, **filters, &block|
64
+ set_callback(name, stage, **filters) do |job|
65
+ ConditionalCallback.new(job, filters).run(method, &block)
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ module CLI
5
+ class Base < Thor
6
+ include Thor::Actions
7
+
8
+ def self.source_root
9
+ File.expand_path("templates", __dir__)
10
+ end
11
+
12
+ private
13
+
14
+ def mock_redis
15
+ Wayfarer.config.redis.factory = ->(_) { MockRedis.new }
16
+ end
17
+
18
+ def load_environment
19
+ Wayfarer::CLI::Runner.loader.setup
20
+ end
21
+
22
+ def free_agent_pool
23
+ Wayfarer::Networking::Pool.instance.free
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ module CLI
5
+ class Generate < Base
6
+ desc "project NAME",
7
+ "Generate new project directory NAME"
8
+ def project(name)
9
+ @name = name
10
+
11
+ empty_directory name
12
+ template "Gemfile.tt", "#{name}/Gemfile"
13
+ template "job.rb.tt", "#{name}/app/jobs/#{name}.rb"
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ module CLI
5
+ class Job < Base
6
+ desc "perform JOB URL",
7
+ "Perform JOB with URL"
8
+ option :mock_redis, type: :boolean
9
+ option :batch, type: :string, default: SecureRandom.uuid
10
+ def perform(job, url)
11
+ load_environment
12
+ mock_redis if options[:mock_redis]
13
+
14
+ url = Addressable::URI.parse(url)
15
+ job = job.classify.constantize.new
16
+ task = Wayfarer::Task.new(url, options[:batch])
17
+ job.arguments.push(task)
18
+ job.perform(task)
19
+ GC.new(job).run
20
+ end
21
+
22
+ desc "enqueue JOB URL",
23
+ "Enqueue JOB with URL"
24
+ option :batch, type: :string, default: SecureRandom.uuid
25
+ def enqueue(job, url)
26
+ load_environment
27
+ mock_redis if options[:mock_redis]
28
+
29
+ url = Addressable::URI.parse(url)
30
+ job = job.classify.constantize
31
+ job.crawl_later(url, batch: options[:batch])
32
+ end
33
+
34
+ desc "execute JOB URL",
35
+ "Execute JOB with async adapter"
36
+ option :mock_redis, type: :boolean
37
+ option :batch, type: :string, default: SecureRandom.uuid
38
+ option :min_threads, type: :numeric, default: 1
39
+ option :max_threads, type: :numeric, default: 1
40
+ def execute(job, url)
41
+ load_environment
42
+ mock_redis if options[:mock_redis]
43
+
44
+ url = Addressable::URI.parse(url)
45
+ job = job.classify.constantize
46
+
47
+ job.queue_adapter = ActiveJob::QueueAdapters::AsyncAdapter.new(min_threads: options[:min_threads],
48
+ max_threads: options[:max_threads])
49
+ scheduler = job.queue_adapter.instance_variable_get(:@scheduler)
50
+ executor = scheduler.instance_variable_get(:@async_executor)
51
+
52
+ job.crawl_later(url, batch: options[:batch])
53
+
54
+ sleep(1) while executor.scheduled_task_count > executor.completed_task_count
55
+
56
+ free_agent_pool
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ # # frozen_string_literal: true
4
+
5
+ module Wayfarer
6
+ module CLI
7
+ class Route < Base
8
+ desc "result JOB URL",
9
+ "Invoke JOB's router with URL"
10
+ def result(job, url)
11
+ load_environment
12
+ url = Addressable::URI.parse(url)
13
+ job = job.classify.constantize
14
+ job.router.invoke(url, job.new.steer)
15
+ say Wayfarer::Routing::PathFinder.result(job.router.root, url)
16
+ end
17
+
18
+ desc "tree JOB URL",
19
+ "Visualize JOB's routing tree for URL"
20
+ def tree(job, url)
21
+ load_environment
22
+ url = Addressable::URI.parse(url)
23
+ job = job.classify.constantize
24
+ job.router.invoke(url, job.new.steer)
25
+ Wayfarer::CLI::RoutePrinter.print(job.router.root, url)
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,116 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ module CLI
5
+ class RoutePrinter < Thor::Shell::Color
6
+ attr_reader :url,
7
+ :path_finder,
8
+ :output
9
+
10
+ INDENT = " "
11
+ REGULAR_SEGMENT = "│ "
12
+ JUNCTION_SEGMENT = "├──"
13
+ CORNER_SEGMENT = "└──"
14
+
15
+ def self.print(route, url)
16
+ route.accept(new(url))
17
+ end
18
+
19
+ def initialize(url)
20
+ @url = url
21
+ @path_finder = Wayfarer::Routing::PathFinder.new(url)
22
+ super()
23
+ end
24
+
25
+ def visit(route)
26
+ route.accept(path_finder) unless route.parent
27
+ return true if route.is_a?(Wayfarer::Routing::RootRoute)
28
+
29
+ puts [segments(route), label(route)].join("")[3..]
30
+ true
31
+ end
32
+
33
+ def segments(route)
34
+ current = segment(route)
35
+ parents = parents(route).map { |parent| parent_segment(parent) }
36
+ [parents, current].join
37
+ end
38
+
39
+ def parent_segment(parent)
40
+ if trailer?(parent)
41
+ INDENT
42
+ else
43
+ REGULAR_SEGMENT
44
+ end
45
+ end
46
+
47
+ def segment(route)
48
+ if trailer?(route)
49
+ CORNER_SEGMENT
50
+ else
51
+ JUNCTION_SEGMENT
52
+ end
53
+ end
54
+
55
+ def label(route)
56
+ [highlight_matcher(route, matcher_label(route)),
57
+ highlight_options(route, options(route)),
58
+ highlight_options(route, params(route))].compact.join(" ")
59
+ end
60
+
61
+ def highlight_matcher(route, string)
62
+ if path_finder.path.include?(route)
63
+ set_color(string, :green, :bold)
64
+ elsif route.matcher.match(url)
65
+ set_color(string, :green)
66
+ else
67
+ set_color(string, :red)
68
+ end
69
+ end
70
+
71
+ def highlight_options(route, string)
72
+ return string unless path_finder.path.include?(route)
73
+
74
+ set_color(string, :green, :bold)
75
+ end
76
+
77
+ def matcher_label(route)
78
+ return "Target" if route.is_a?(Wayfarer::Routing::TargetRoute)
79
+
80
+ route.matcher.class.name.demodulize
81
+ end
82
+
83
+ def options(route)
84
+ return "" if route.is_a?(Wayfarer::Routing::RootRoute)
85
+
86
+ case (matcher = route.matcher)
87
+ when Wayfarer::Routing::Matchers::Host then matcher.host
88
+ when Wayfarer::Routing::Matchers::Path then matcher.path
89
+ when Wayfarer::Routing::Matchers::Query then matcher.fields
90
+ when Wayfarer::Routing::Matchers::Custom then "##{route.action}"
91
+ when Wayfarer::Routing::Matchers::Scheme then matcher.scheme
92
+ when Wayfarer::Routing::Matchers::Suffix then matcher.suffix
93
+ end
94
+ end
95
+
96
+ def params(route)
97
+ params = route.matcher.params(url)
98
+ "=> #{params.symbolize_keys}" if params.any?
99
+ end
100
+
101
+ private
102
+
103
+ def parents(route, current = [])
104
+ return current unless route.parent
105
+
106
+ parents(route.parent, [route.parent, *current])
107
+ end
108
+
109
+ def trailer?(route)
110
+ return true unless route.parent
111
+
112
+ route.parent.children.last == route
113
+ end
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ module CLI
5
+ class Runner < Thor
6
+ def self.exit_on_failure?
7
+ true
8
+ end
9
+
10
+ def self.loader
11
+ # TODO: Raises if $PWD/app/jobs does not exist
12
+
13
+ @loader ||= Zeitwerk::Loader.new.tap do |load|
14
+ load.inflector = Zeitwerk::GemInflector.new(__FILE__)
15
+ load.push_dir("#{Dir.pwd}/app/jobs")
16
+ end
17
+ end
18
+
19
+ desc "version", "Print version"
20
+ def version
21
+ say Wayfarer::VERSION::STRING
22
+ end
23
+
24
+ desc "route", "Routing"
25
+ subcommand "route", Wayfarer::CLI::Route
26
+
27
+ desc "generate", "Generators"
28
+ subcommand "generate", Wayfarer::CLI::Generate
29
+
30
+ desc "job", "Jobs"
31
+ subcommand "job", Wayfarer::CLI::Job
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "wayfarer", <%= Wayfarer::VERSION::STRING.inspect %>
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ class <%= @name.camelize %> < Wayfarer::Base
4
+ route { to :index }
5
+
6
+ retry_on ConnectionPool::TimeoutError, attempts: 3
7
+
8
+ def index
9
+ end
10
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ module Config
5
+ Capybara = Struct.new(driver: {
6
+ env_key: "WAYFARER_CAPYBARA_DRIVER",
7
+ type: Symbol
8
+ })
9
+ end
10
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ module Config
5
+ Ferrum = Struct.new(options: {
6
+ env_key: "WAYFARER_FERRUM_OPTIONS",
7
+ type: Hash,
8
+ default: {}
9
+ })
10
+ end
11
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ module Config
5
+ Networking = Struct.new(agent: {
6
+ env_key: "WAYFARER_NETWORK_AGENT",
7
+ type: Symbol,
8
+ default: :http
9
+ },
10
+ pool_size: {
11
+ env_key: "WAYFARER_NETWORK_POOL_SIZE",
12
+ type: Integer,
13
+ default: 1
14
+ },
15
+ pool_timeout: {
16
+ env_key: "WAYFARER_NETWORK_POOL_TIMEOUT",
17
+ type: Integer,
18
+ default: 10
19
+ },
20
+ http_headers: {
21
+ env_key: "WAYFARER_NETWORK_HTTP_HEADERS",
22
+ type: Hash,
23
+ default: {}
24
+ })
25
+ end
26
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ module Config
5
+ Redis = Struct.new(url: {
6
+ env_key: "WAYFARER_REDIS_URL",
7
+ type: String,
8
+ default: "redis://localhost:6379"
9
+ },
10
+ factory: {
11
+ default: ->(redis) { ::Redis.new(url: redis.url) }
12
+ })
13
+ end
14
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ module Config
5
+ Root = Struct.new(ferrum: { default: Wayfarer::Config::Ferrum.new },
6
+ network: { default: Wayfarer::Config::Networking.new },
7
+ redis: { default: Wayfarer::Config::Redis.new },
8
+ selenium: { default: Wayfarer::Config::Selenium.new },
9
+ capybara: { default: Wayfarer::Config::Capybara.new })
10
+ end
11
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ module Config
5
+ Selenium = Struct.new(driver: {
6
+ env_key: "WAYFARER_SELENIUM_DRIVER",
7
+ type: Symbol,
8
+ default: :chrome
9
+ },
10
+ options: {
11
+ env_key: "WAYFARER_SELENIUM_OPTIONS",
12
+ type: Hash,
13
+ default: {}
14
+ },
15
+ client_timeout: {
16
+ env_key: "WAYFARER_SELENIUM_CLIENT_TIMEOUT",
17
+ type: Integer,
18
+ default: 60 # seconds
19
+ })
20
+ end
21
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wayfarer
4
+ module Config
5
+ module Strconv
6
+ module_function
7
+
8
+ def parse(str, type = nil)
9
+ return primitive(str) unless type
10
+
11
+ case type.name
12
+ when "Hash" then hash(str)
13
+ when "Array" then array(str)
14
+ when "Symbol" then str.to_sym
15
+ when "Integer" then Integer(str)
16
+ else str
17
+ end
18
+ end
19
+
20
+ def hash(str)
21
+ array(str).reduce({}) do |acc, pair|
22
+ k, v = pair.split(":", 2)
23
+ next acc unless k && v
24
+
25
+ acc.merge({ parse(k, Symbol) => primitive(v) })
26
+ end
27
+ end
28
+
29
+ def array(str)
30
+ str.split(",").map(&:strip)
31
+ end
32
+
33
+ def primitive(str)
34
+ return true if str == "true"
35
+ return false if str == "false"
36
+
37
+ begin
38
+ parse(str, Integer)
39
+ rescue StandardError
40
+ str
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end