wayfarer 0.0.3 → 0.4.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (369) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/ci.yaml +32 -0
  3. data/.gitignore +3 -4
  4. data/.rubocop.yml +25 -9
  5. data/.ruby-version +1 -1
  6. data/Dockerfile +5 -0
  7. data/Gemfile +1 -7
  8. data/Gemfile.lock +221 -0
  9. data/RELEASING.md +17 -0
  10. data/Rakefile +38 -90
  11. data/bin/wayfarer +1 -111
  12. data/docker-compose.yml +32 -0
  13. data/docs/cookbook/batch_routing.md +22 -0
  14. data/docs/cookbook/consent_screen.md +36 -0
  15. data/docs/cookbook/executing_javascript.md +41 -0
  16. data/docs/cookbook/querying_html.md +42 -0
  17. data/docs/cookbook/screenshots.md +27 -0
  18. data/docs/cookbook/user_agent.md +7 -0
  19. data/docs/guides/browser_automation/capybara.md +69 -0
  20. data/docs/guides/browser_automation/custom_adapters.md +100 -0
  21. data/docs/guides/browser_automation/ferrum.md +39 -0
  22. data/docs/guides/browser_automation/selenium.md +63 -0
  23. data/docs/guides/callbacks.md +131 -31
  24. data/docs/guides/configuration.md +24 -169
  25. data/docs/guides/debugging.md +17 -0
  26. data/docs/guides/error_handling.md +30 -45
  27. data/docs/guides/jobs.md +101 -0
  28. data/docs/guides/navigation.md +73 -0
  29. data/docs/guides/networking.md +94 -0
  30. data/docs/guides/pages.md +52 -0
  31. data/docs/guides/performance.md +130 -0
  32. data/docs/guides/reliability.md +41 -0
  33. data/docs/guides/routing/steering.md +30 -0
  34. data/docs/guides/tasks.md +14 -0
  35. data/docs/index.md +40 -66
  36. data/docs/reference/api/base.md +48 -0
  37. data/docs/reference/api/route.md +182 -0
  38. data/docs/reference/cli.md +61 -0
  39. data/docs/reference/configuration_keys.md +42 -0
  40. data/docs/reference/environment_variables.md +83 -0
  41. data/lib/wayfarer/base.rb +50 -0
  42. data/lib/wayfarer/callbacks.rb +71 -0
  43. data/lib/wayfarer/cli/base.rb +27 -0
  44. data/lib/wayfarer/cli/generate.rb +17 -0
  45. data/lib/wayfarer/cli/job.rb +60 -0
  46. data/lib/wayfarer/cli/route.rb +29 -0
  47. data/lib/wayfarer/cli/route_printer.rb +116 -0
  48. data/lib/wayfarer/cli/runner.rb +34 -0
  49. data/lib/wayfarer/cli/templates/Gemfile.tt +5 -0
  50. data/lib/wayfarer/cli/templates/job.rb.tt +10 -0
  51. data/lib/wayfarer/config/capybara.rb +10 -0
  52. data/lib/wayfarer/config/ferrum.rb +11 -0
  53. data/lib/wayfarer/config/networking.rb +26 -0
  54. data/lib/wayfarer/config/redis.rb +14 -0
  55. data/lib/wayfarer/config/root.rb +11 -0
  56. data/lib/wayfarer/config/selenium.rb +21 -0
  57. data/lib/wayfarer/config/strconv.rb +45 -0
  58. data/lib/wayfarer/config/struct.rb +72 -0
  59. data/lib/wayfarer/gc.rb +15 -0
  60. data/lib/wayfarer/middleware/chain.rb +19 -0
  61. data/lib/wayfarer/middleware/dedup.rb +25 -0
  62. data/lib/wayfarer/middleware/fetch.rb +47 -0
  63. data/lib/wayfarer/middleware/normalize.rb +25 -0
  64. data/lib/wayfarer/middleware/router.rb +53 -0
  65. data/lib/wayfarer/middleware/stage.rb +23 -0
  66. data/lib/wayfarer/middleware/worker.rb +30 -0
  67. data/lib/wayfarer/networking/capybara.rb +28 -0
  68. data/lib/wayfarer/networking/context.rb +36 -0
  69. data/lib/wayfarer/networking/ferrum.rb +35 -0
  70. data/lib/wayfarer/networking/http.rb +34 -0
  71. data/lib/wayfarer/networking/pool.rb +40 -0
  72. data/lib/wayfarer/networking/result.rb +18 -0
  73. data/lib/wayfarer/networking/selenium.rb +43 -0
  74. data/lib/wayfarer/networking/strategy.rb +38 -0
  75. data/lib/wayfarer/page.rb +17 -74
  76. data/lib/wayfarer/parsing/json.rb +17 -0
  77. data/lib/wayfarer/parsing/xml.rb +17 -0
  78. data/lib/wayfarer/redis/.#barrier.rb +1 -0
  79. data/lib/wayfarer/redis/barrier.rb +36 -0
  80. data/lib/wayfarer/redis/connection.rb +13 -0
  81. data/lib/wayfarer/redis/counter.rb +29 -0
  82. data/lib/wayfarer/redis/pool.rb +20 -0
  83. data/lib/wayfarer/redis/version.rb +19 -0
  84. data/lib/wayfarer/routing/dsl.rb +57 -0
  85. data/lib/wayfarer/routing/matchers/custom.rb +25 -0
  86. data/lib/wayfarer/routing/matchers/host.rb +19 -0
  87. data/lib/wayfarer/routing/matchers/path.rb +49 -0
  88. data/lib/wayfarer/routing/matchers/query.rb +63 -0
  89. data/lib/wayfarer/routing/matchers/scheme.rb +17 -0
  90. data/lib/wayfarer/routing/matchers/suffix.rb +17 -0
  91. data/lib/wayfarer/routing/matchers/url.rb +17 -0
  92. data/lib/wayfarer/routing/path_finder.rb +46 -0
  93. data/lib/wayfarer/routing/result.rb +15 -0
  94. data/lib/wayfarer/routing/root_route.rb +7 -0
  95. data/lib/wayfarer/routing/route.rb +47 -0
  96. data/lib/wayfarer/routing/router.rb +10 -54
  97. data/lib/wayfarer/routing/target_route.rb +7 -0
  98. data/lib/wayfarer/serializer.rb +17 -0
  99. data/lib/wayfarer/stringify.rb +47 -0
  100. data/lib/wayfarer/task.rb +34 -0
  101. data/lib/wayfarer.rb +48 -57
  102. data/mkdocs.yml +47 -0
  103. data/requirements.txt +1 -0
  104. data/spec/base_spec.rb +233 -0
  105. data/spec/callbacks_spec.rb +102 -0
  106. data/spec/cli/generate_spec.rb +39 -0
  107. data/spec/cli/job_spec.rb +74 -0
  108. data/spec/cli/version_spec.rb +13 -0
  109. data/spec/config/capybara_spec.rb +18 -0
  110. data/spec/config/ferrum_spec.rb +24 -0
  111. data/spec/config/networking_spec.rb +73 -0
  112. data/spec/config/redis_spec.rb +32 -0
  113. data/spec/config/root_spec.rb +31 -0
  114. data/spec/config/selenium_spec.rb +56 -0
  115. data/spec/config/strconv_spec.rb +58 -0
  116. data/spec/config/struct_spec.rb +66 -0
  117. data/spec/factories/middleware.rb +15 -0
  118. data/spec/factories/page.rb +78 -0
  119. data/spec/factories/task.rb +12 -0
  120. data/spec/fixtures/dummy_job.rb +7 -0
  121. data/spec/gc_spec.rb +63 -0
  122. data/spec/middleware/chain_spec.rb +96 -0
  123. data/spec/middleware/dedup_spec.rb +76 -0
  124. data/spec/middleware/fetch_spec.rb +100 -0
  125. data/spec/middleware/normalize_spec.rb +28 -0
  126. data/spec/middleware/router_spec.rb +80 -0
  127. data/spec/middleware/stage_spec.rb +39 -0
  128. data/spec/middleware/worker_spec.rb +117 -0
  129. data/spec/networking/capybara_spec.rb +12 -0
  130. data/spec/networking/context_spec.rb +127 -0
  131. data/spec/networking/ferrum_spec.rb +12 -0
  132. data/spec/networking/http_spec.rb +12 -0
  133. data/spec/networking/pool_spec.rb +67 -0
  134. data/spec/networking/selenium_spec.rb +12 -0
  135. data/spec/networking/strategy.rb +170 -0
  136. data/spec/page_spec.rb +21 -12
  137. data/spec/{parsers/json_parser_spec.rb → parsing/json_spec.rb} +5 -4
  138. data/spec/{parsers/xml_parser_spec.rb → parsing/xml_spec.rb} +3 -2
  139. data/spec/redis/barrier_spec.rb +78 -0
  140. data/spec/redis/counter_spec.rb +32 -0
  141. data/spec/redis/pool_spec.rb +18 -0
  142. data/spec/redis/version_spec.rb +13 -0
  143. data/spec/routing/dsl_spec.rb +98 -0
  144. data/spec/routing/integration_spec.rb +110 -0
  145. data/spec/routing/matchers/custom_spec.rb +31 -0
  146. data/spec/routing/matchers/host_spec.rb +49 -0
  147. data/spec/routing/matchers/path_spec.rb +43 -0
  148. data/spec/routing/matchers/query_spec.rb +137 -0
  149. data/spec/routing/matchers/scheme_spec.rb +25 -0
  150. data/spec/routing/{filetypes_rule_spec.rb → matchers/suffix_spec.rb} +14 -13
  151. data/spec/routing/matchers/uri_spec.rb +27 -0
  152. data/spec/routing/path_finder_spec.rb +33 -0
  153. data/spec/routing/root_route_spec.rb +29 -0
  154. data/spec/routing/route_spec.rb +43 -0
  155. data/spec/routing/router_spec.rb +13 -56
  156. data/spec/spec_helpers.rb +73 -38
  157. data/spec/stringify_spec.rb +23 -0
  158. data/{support → spec/support}/static/finders.html +0 -0
  159. data/{support → spec/support}/static/graph/details/a.html +0 -0
  160. data/{support → spec/support}/static/graph/details/b.html +0 -0
  161. data/{support → spec/support}/static/graph/index.html +0 -0
  162. data/{support → spec/support}/static/json/dummy.json +0 -0
  163. data/{support → spec/support}/static/links/links.html +0 -0
  164. data/{support → spec/support}/static/xml/dummy.xml +0 -0
  165. data/{support → spec/support}/test_app.rb +9 -2
  166. data/spec/task_spec.rb +27 -0
  167. data/spec/wayfarer_spec.rb +2 -13
  168. data/wayfarer.gemspec +40 -42
  169. metadata +234 -361
  170. data/.travis.yml +0 -5
  171. data/Changelog.md +0 -10
  172. data/README.md +0 -21
  173. data/benchmark/frontiers.rb +0 -143
  174. data/docs/.gitignore +0 -2
  175. data/docs/_config.yml +0 -15
  176. data/docs/_includes/base.html +0 -7
  177. data/docs/_includes/head.html +0 -10
  178. data/docs/_includes/navigation.html +0 -187
  179. data/docs/_layouts/default.html +0 -42
  180. data/docs/_sass/base.scss +0 -439
  181. data/docs/_sass/variables.scss +0 -24
  182. data/docs/_sass/vendor/bourbon/_bourbon-deprecate.scss +0 -19
  183. data/docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss +0 -425
  184. data/docs/_sass/vendor/bourbon/_bourbon.scss +0 -90
  185. data/docs/_sass/vendor/bourbon/addons/_border-color.scss +0 -29
  186. data/docs/_sass/vendor/bourbon/addons/_border-radius.scss +0 -48
  187. data/docs/_sass/vendor/bourbon/addons/_border-style.scss +0 -28
  188. data/docs/_sass/vendor/bourbon/addons/_border-width.scss +0 -28
  189. data/docs/_sass/vendor/bourbon/addons/_buttons.scss +0 -69
  190. data/docs/_sass/vendor/bourbon/addons/_clearfix.scss +0 -25
  191. data/docs/_sass/vendor/bourbon/addons/_ellipsis.scss +0 -30
  192. data/docs/_sass/vendor/bourbon/addons/_font-stacks.scss +0 -31
  193. data/docs/_sass/vendor/bourbon/addons/_hide-text.scss +0 -27
  194. data/docs/_sass/vendor/bourbon/addons/_margin.scss +0 -29
  195. data/docs/_sass/vendor/bourbon/addons/_padding.scss +0 -29
  196. data/docs/_sass/vendor/bourbon/addons/_position.scss +0 -51
  197. data/docs/_sass/vendor/bourbon/addons/_prefixer.scss +0 -66
  198. data/docs/_sass/vendor/bourbon/addons/_retina-image.scss +0 -27
  199. data/docs/_sass/vendor/bourbon/addons/_size.scss +0 -56
  200. data/docs/_sass/vendor/bourbon/addons/_text-inputs.scss +0 -118
  201. data/docs/_sass/vendor/bourbon/addons/_timing-functions.scss +0 -34
  202. data/docs/_sass/vendor/bourbon/addons/_triangle.scss +0 -63
  203. data/docs/_sass/vendor/bourbon/addons/_word-wrap.scss +0 -29
  204. data/docs/_sass/vendor/bourbon/css3/_animation.scss +0 -61
  205. data/docs/_sass/vendor/bourbon/css3/_appearance.scss +0 -5
  206. data/docs/_sass/vendor/bourbon/css3/_backface-visibility.scss +0 -5
  207. data/docs/_sass/vendor/bourbon/css3/_background-image.scss +0 -44
  208. data/docs/_sass/vendor/bourbon/css3/_background.scss +0 -57
  209. data/docs/_sass/vendor/bourbon/css3/_border-image.scss +0 -61
  210. data/docs/_sass/vendor/bourbon/css3/_calc.scss +0 -6
  211. data/docs/_sass/vendor/bourbon/css3/_columns.scss +0 -67
  212. data/docs/_sass/vendor/bourbon/css3/_filter.scss +0 -6
  213. data/docs/_sass/vendor/bourbon/css3/_flex-box.scss +0 -327
  214. data/docs/_sass/vendor/bourbon/css3/_font-face.scss +0 -29
  215. data/docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss +0 -6
  216. data/docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss +0 -12
  217. data/docs/_sass/vendor/bourbon/css3/_hyphens.scss +0 -6
  218. data/docs/_sass/vendor/bourbon/css3/_image-rendering.scss +0 -15
  219. data/docs/_sass/vendor/bourbon/css3/_keyframes.scss +0 -38
  220. data/docs/_sass/vendor/bourbon/css3/_linear-gradient.scss +0 -40
  221. data/docs/_sass/vendor/bourbon/css3/_perspective.scss +0 -12
  222. data/docs/_sass/vendor/bourbon/css3/_placeholder.scss +0 -10
  223. data/docs/_sass/vendor/bourbon/css3/_radial-gradient.scss +0 -40
  224. data/docs/_sass/vendor/bourbon/css3/_selection.scss +0 -44
  225. data/docs/_sass/vendor/bourbon/css3/_text-decoration.scss +0 -27
  226. data/docs/_sass/vendor/bourbon/css3/_transform.scss +0 -21
  227. data/docs/_sass/vendor/bourbon/css3/_transition.scss +0 -81
  228. data/docs/_sass/vendor/bourbon/css3/_user-select.scss +0 -5
  229. data/docs/_sass/vendor/bourbon/functions/_assign-inputs.scss +0 -16
  230. data/docs/_sass/vendor/bourbon/functions/_contains-falsy.scss +0 -25
  231. data/docs/_sass/vendor/bourbon/functions/_contains.scss +0 -31
  232. data/docs/_sass/vendor/bourbon/functions/_is-length.scss +0 -16
  233. data/docs/_sass/vendor/bourbon/functions/_is-light.scss +0 -26
  234. data/docs/_sass/vendor/bourbon/functions/_is-number.scss +0 -16
  235. data/docs/_sass/vendor/bourbon/functions/_is-size.scss +0 -23
  236. data/docs/_sass/vendor/bourbon/functions/_modular-scale.scss +0 -74
  237. data/docs/_sass/vendor/bourbon/functions/_px-to-em.scss +0 -24
  238. data/docs/_sass/vendor/bourbon/functions/_px-to-rem.scss +0 -26
  239. data/docs/_sass/vendor/bourbon/functions/_shade.scss +0 -24
  240. data/docs/_sass/vendor/bourbon/functions/_strip-units.scss +0 -22
  241. data/docs/_sass/vendor/bourbon/functions/_tint.scss +0 -24
  242. data/docs/_sass/vendor/bourbon/functions/_transition-property-name.scss +0 -37
  243. data/docs/_sass/vendor/bourbon/functions/_unpack.scss +0 -32
  244. data/docs/_sass/vendor/bourbon/helpers/_convert-units.scss +0 -26
  245. data/docs/_sass/vendor/bourbon/helpers/_directional-values.scss +0 -108
  246. data/docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss +0 -53
  247. data/docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss +0 -24
  248. data/docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss +0 -35
  249. data/docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss +0 -51
  250. data/docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss +0 -77
  251. data/docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss +0 -41
  252. data/docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss +0 -74
  253. data/docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss +0 -55
  254. data/docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss +0 -28
  255. data/docs/_sass/vendor/bourbon/helpers/_render-gradients.scss +0 -31
  256. data/docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss +0 -15
  257. data/docs/_sass/vendor/bourbon/helpers/_str-to-num.scss +0 -55
  258. data/docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss +0 -7
  259. data/docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss +0 -8
  260. data/docs/_sass/vendor/bourbon/settings/_prefixer.scss +0 -9
  261. data/docs/_sass/vendor/bourbon/settings/_px-to-em.scss +0 -1
  262. data/docs/_sass/vendor/neat/_neat-helpers.scss +0 -11
  263. data/docs/_sass/vendor/neat/_neat.scss +0 -23
  264. data/docs/_sass/vendor/neat/functions/_new-breakpoint.scss +0 -49
  265. data/docs/_sass/vendor/neat/functions/_private.scss +0 -114
  266. data/docs/_sass/vendor/neat/grid/_box-sizing.scss +0 -15
  267. data/docs/_sass/vendor/neat/grid/_direction-context.scss +0 -33
  268. data/docs/_sass/vendor/neat/grid/_display-context.scss +0 -28
  269. data/docs/_sass/vendor/neat/grid/_fill-parent.scss +0 -22
  270. data/docs/_sass/vendor/neat/grid/_media.scss +0 -92
  271. data/docs/_sass/vendor/neat/grid/_omega.scss +0 -87
  272. data/docs/_sass/vendor/neat/grid/_outer-container.scss +0 -34
  273. data/docs/_sass/vendor/neat/grid/_pad.scss +0 -25
  274. data/docs/_sass/vendor/neat/grid/_private.scss +0 -35
  275. data/docs/_sass/vendor/neat/grid/_row.scss +0 -52
  276. data/docs/_sass/vendor/neat/grid/_shift.scss +0 -50
  277. data/docs/_sass/vendor/neat/grid/_span-columns.scss +0 -94
  278. data/docs/_sass/vendor/neat/grid/_to-deprecate.scss +0 -97
  279. data/docs/_sass/vendor/neat/grid/_visual-grid.scss +0 -42
  280. data/docs/_sass/vendor/neat/mixins/_clearfix.scss +0 -25
  281. data/docs/_sass/vendor/neat/settings/_disable-warnings.scss +0 -13
  282. data/docs/_sass/vendor/neat/settings/_grid.scss +0 -51
  283. data/docs/_sass/vendor/neat/settings/_visual-grid.scss +0 -27
  284. data/docs/_sass/vendor/normalize-3.0.2.scss +0 -427
  285. data/docs/_sass/vendor/pygments.scss +0 -356
  286. data/docs/automating_browsers/capybara.md +0 -70
  287. data/docs/css/screen.scss +0 -7
  288. data/docs/guides/cli.md +0 -52
  289. data/docs/guides/frontiers.md +0 -93
  290. data/docs/guides/halting.md +0 -23
  291. data/docs/guides/job_queues.md +0 -26
  292. data/docs/guides/locals.md +0 -36
  293. data/docs/guides/logging.md +0 -22
  294. data/docs/guides/page_objects.md +0 -67
  295. data/docs/guides/peeking.md +0 -46
  296. data/docs/guides/selenium_capybara.md +0 -100
  297. data/docs/guides/tutorial.md +0 -452
  298. data/docs/js/navigation.js +0 -11
  299. data/docs/misc/contributing.md +0 -20
  300. data/docs/misc/testing.md +0 -11
  301. data/docs/recipes/authentication.md +0 -23
  302. data/docs/recipes/csv.md +0 -29
  303. data/docs/recipes/javascript.md +0 -20
  304. data/docs/recipes/multiple_uris.md +0 -18
  305. data/docs/recipes/screenshots.md +0 -20
  306. data/docs/routing/custom_rules.md +0 -16
  307. data/docs/routing/filetypes_rules.md +0 -21
  308. data/docs/routing/host_rules.md +0 -24
  309. data/docs/routing/path_rules.md +0 -33
  310. data/docs/routing/protocol_rules.md +0 -17
  311. data/docs/routing/query_rules.md +0 -69
  312. data/docs/routing/routes.md +0 -96
  313. data/docs/routing/uri_rules.md +0 -18
  314. data/examples/collect_github_issues.rb +0 -65
  315. data/examples/find_foobar_on_wikipedia.rb +0 -23
  316. data/lib/wayfarer/configuration.rb +0 -86
  317. data/lib/wayfarer/crawl.rb +0 -79
  318. data/lib/wayfarer/crawl_observer.rb +0 -103
  319. data/lib/wayfarer/dispatcher.rb +0 -104
  320. data/lib/wayfarer/finders.rb +0 -61
  321. data/lib/wayfarer/frontiers/frontier.rb +0 -79
  322. data/lib/wayfarer/frontiers/memory_bloomfilter.rb +0 -32
  323. data/lib/wayfarer/frontiers/memory_frontier.rb +0 -76
  324. data/lib/wayfarer/frontiers/memory_trie_frontier.rb +0 -39
  325. data/lib/wayfarer/frontiers/normalize_uris.rb +0 -48
  326. data/lib/wayfarer/frontiers/redis_bloomfilter.rb +0 -34
  327. data/lib/wayfarer/frontiers/redis_frontier.rb +0 -83
  328. data/lib/wayfarer/http_adapters/adapter_pool.rb +0 -62
  329. data/lib/wayfarer/http_adapters/net_http_adapter.rb +0 -77
  330. data/lib/wayfarer/http_adapters/selenium_adapter.rb +0 -80
  331. data/lib/wayfarer/job.rb +0 -211
  332. data/lib/wayfarer/locals.rb +0 -40
  333. data/lib/wayfarer/parsers/json_parser.rb +0 -20
  334. data/lib/wayfarer/parsers/xml_parser.rb +0 -27
  335. data/lib/wayfarer/processor.rb +0 -103
  336. data/lib/wayfarer/routing/custom_rule.rb +0 -21
  337. data/lib/wayfarer/routing/filetypes_rule.rb +0 -20
  338. data/lib/wayfarer/routing/host_rule.rb +0 -19
  339. data/lib/wayfarer/routing/path_rule.rb +0 -54
  340. data/lib/wayfarer/routing/protocol_rule.rb +0 -21
  341. data/lib/wayfarer/routing/query_rule.rb +0 -59
  342. data/lib/wayfarer/routing/rule.rb +0 -114
  343. data/lib/wayfarer/routing/uri_rule.rb +0 -21
  344. data/spec/configuration_spec.rb +0 -26
  345. data/spec/crawl_spec.rb +0 -48
  346. data/spec/finders_spec.rb +0 -49
  347. data/spec/frontiers/memory_bloomfilter_spec.rb +0 -6
  348. data/spec/frontiers/memory_frontier_spec.rb +0 -6
  349. data/spec/frontiers/memory_trie_frontier_spec.rb +0 -6
  350. data/spec/frontiers/normalize_uris_spec.rb +0 -59
  351. data/spec/frontiers/redis_bloomfilter_spec.rb +0 -6
  352. data/spec/frontiers/redis_frontier_spec.rb +0 -6
  353. data/spec/http_adapters/adapter_pool_spec.rb +0 -33
  354. data/spec/http_adapters/net_http_adapter_spec.rb +0 -83
  355. data/spec/http_adapters/selenium_adapter_spec.rb +0 -53
  356. data/spec/integration/callbacks_spec.rb +0 -42
  357. data/spec/integration/locals_spec.rb +0 -106
  358. data/spec/integration/peeking_spec.rb +0 -61
  359. data/spec/job_spec.rb +0 -122
  360. data/spec/processor_spec.rb +0 -31
  361. data/spec/routing/custom_rule_spec.rb +0 -26
  362. data/spec/routing/host_rule_spec.rb +0 -48
  363. data/spec/routing/path_rule_spec.rb +0 -66
  364. data/spec/routing/protocol_rule_spec.rb +0 -26
  365. data/spec/routing/query_rule_spec.rb +0 -124
  366. data/spec/routing/rule_spec.rb +0 -251
  367. data/spec/routing/uri_rule_spec.rb +0 -24
  368. data/spec/shared/frontier.rb +0 -96
  369. data/wayfarer-jruby.gemspec +0 -49
metadata CHANGED
@@ -1,309 +1,287 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: wayfarer
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.3
4
+ version: 0.4.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dominic Bauer
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-05-31 00:00:00.000000000 Z
11
+ date: 2021-09-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: connection_pool
14
+ name: activejob
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - "~>"
17
+ - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: '2.2'
19
+ version: '6.0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - "~>"
24
+ - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: '2.2'
26
+ version: '6.0'
27
27
  - !ruby/object:Gem::Dependency
28
- name: nokogiri
28
+ name: addressable
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '1.6'
33
+ version: '2.8'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '1.6'
40
+ version: '2.8'
41
41
  - !ruby/object:Gem::Dependency
42
- name: selenium-webdriver
42
+ name: capybara
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '3.4'
47
+ version: '3.0'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '3.4'
55
- - !ruby/object:Gem::Dependency
56
- name: selenium-emulated_features
57
- requirement: !ruby/object:Gem::Requirement
58
- requirements:
59
- - - '='
60
- - !ruby/object:Gem::Version
61
- version: 2.0.1
62
- type: :runtime
63
- prerelease: false
64
- version_requirements: !ruby/object:Gem::Requirement
65
- requirements:
66
- - - '='
67
- - !ruby/object:Gem::Version
68
- version: 2.0.1
54
+ version: '3.0'
69
55
  - !ruby/object:Gem::Dependency
70
- name: net-http-persistent
56
+ name: connection_pool
71
57
  requirement: !ruby/object:Gem::Requirement
72
58
  requirements:
73
59
  - - "~>"
74
60
  - !ruby/object:Gem::Version
75
- version: '2.9'
61
+ version: '2.2'
76
62
  type: :runtime
77
63
  prerelease: false
78
64
  version_requirements: !ruby/object:Gem::Requirement
79
65
  requirements:
80
66
  - - "~>"
81
67
  - !ruby/object:Gem::Version
82
- version: '2.9'
68
+ version: '2.2'
83
69
  - !ruby/object:Gem::Dependency
84
- name: mime-types
70
+ name: docile
85
71
  requirement: !ruby/object:Gem::Requirement
86
72
  requirements:
87
73
  - - "~>"
88
74
  - !ruby/object:Gem::Version
89
- version: '2.6'
75
+ version: '1.1'
90
76
  type: :runtime
91
77
  prerelease: false
92
78
  version_requirements: !ruby/object:Gem::Requirement
93
79
  requirements:
94
80
  - - "~>"
95
81
  - !ruby/object:Gem::Version
96
- version: '2.6'
82
+ version: '1.1'
97
83
  - !ruby/object:Gem::Dependency
98
- name: pismo
84
+ name: ferrum
99
85
  requirement: !ruby/object:Gem::Requirement
100
86
  requirements:
101
87
  - - "~>"
102
88
  - !ruby/object:Gem::Version
103
- version: '0.7'
89
+ version: '0.9'
104
90
  type: :runtime
105
91
  prerelease: false
106
92
  version_requirements: !ruby/object:Gem::Requirement
107
93
  requirements:
108
94
  - - "~>"
109
95
  - !ruby/object:Gem::Version
110
- version: '0.7'
96
+ version: '0.9'
111
97
  - !ruby/object:Gem::Dependency
112
- name: mustermann
98
+ name: metainspector
113
99
  requirement: !ruby/object:Gem::Requirement
114
100
  requirements:
115
101
  - - "~>"
116
102
  - !ruby/object:Gem::Version
117
- version: '0.4'
103
+ version: '5.0'
118
104
  type: :runtime
119
105
  prerelease: false
120
106
  version_requirements: !ruby/object:Gem::Requirement
121
107
  requirements:
122
108
  - - "~>"
123
109
  - !ruby/object:Gem::Version
124
- version: '0.4'
110
+ version: '5.0'
125
111
  - !ruby/object:Gem::Dependency
126
- name: activejob
112
+ name: mime-types
127
113
  requirement: !ruby/object:Gem::Requirement
128
114
  requirements:
129
115
  - - "~>"
130
116
  - !ruby/object:Gem::Version
131
- version: '4.2'
117
+ version: '3.0'
132
118
  type: :runtime
133
119
  prerelease: false
134
120
  version_requirements: !ruby/object:Gem::Requirement
135
121
  requirements:
136
122
  - - "~>"
137
123
  - !ruby/object:Gem::Version
138
- version: '4.2'
124
+ version: '3.0'
139
125
  - !ruby/object:Gem::Dependency
140
- name: thor
126
+ name: mustermann
141
127
  requirement: !ruby/object:Gem::Requirement
142
128
  requirements:
143
129
  - - "~>"
144
130
  - !ruby/object:Gem::Version
145
- version: '0.19'
131
+ version: '1.1'
146
132
  type: :runtime
147
133
  prerelease: false
148
134
  version_requirements: !ruby/object:Gem::Requirement
149
135
  requirements:
150
136
  - - "~>"
151
137
  - !ruby/object:Gem::Version
152
- version: '0.19'
138
+ version: '1.1'
153
139
  - !ruby/object:Gem::Dependency
154
- name: activesupport
140
+ name: mock_redis
155
141
  requirement: !ruby/object:Gem::Requirement
156
142
  requirements:
157
143
  - - "~>"
158
144
  - !ruby/object:Gem::Version
159
- version: '4.2'
145
+ version: '0.29'
160
146
  type: :runtime
161
147
  prerelease: false
162
148
  version_requirements: !ruby/object:Gem::Requirement
163
149
  requirements:
164
150
  - - "~>"
165
151
  - !ruby/object:Gem::Version
166
- version: '4.2'
152
+ version: '0.29'
167
153
  - !ruby/object:Gem::Dependency
168
- name: capybara
154
+ name: net-http-persistent
169
155
  requirement: !ruby/object:Gem::Requirement
170
156
  requirements:
171
157
  - - "~>"
172
158
  - !ruby/object:Gem::Version
173
- version: '2.5'
159
+ version: '3.0'
174
160
  type: :runtime
175
161
  prerelease: false
176
162
  version_requirements: !ruby/object:Gem::Requirement
177
163
  requirements:
178
164
  - - "~>"
179
165
  - !ruby/object:Gem::Version
180
- version: '2.5'
166
+ version: '3.0'
181
167
  - !ruby/object:Gem::Dependency
182
- name: chronic
168
+ name: nokogiri
183
169
  requirement: !ruby/object:Gem::Requirement
184
170
  requirements:
185
171
  - - "~>"
186
172
  - !ruby/object:Gem::Version
187
- version: '0.10'
173
+ version: '1.11'
188
174
  type: :runtime
189
175
  prerelease: false
190
176
  version_requirements: !ruby/object:Gem::Requirement
191
177
  requirements:
192
178
  - - "~>"
193
179
  - !ruby/object:Gem::Version
194
- version: '0.10'
180
+ version: '1.11'
195
181
  - !ruby/object:Gem::Dependency
196
- name: redis
182
+ name: normalize_url
197
183
  requirement: !ruby/object:Gem::Requirement
198
184
  requirements:
199
185
  - - "~>"
200
186
  - !ruby/object:Gem::Version
201
- version: '3.2'
187
+ version: 0.0.6
202
188
  type: :runtime
203
189
  prerelease: false
204
190
  version_requirements: !ruby/object:Gem::Requirement
205
191
  requirements:
206
192
  - - "~>"
207
193
  - !ruby/object:Gem::Version
208
- version: '3.2'
194
+ version: 0.0.6
209
195
  - !ruby/object:Gem::Dependency
210
- name: concurrent-ruby
196
+ name: redis
211
197
  requirement: !ruby/object:Gem::Requirement
212
198
  requirements:
213
199
  - - "~>"
214
200
  - !ruby/object:Gem::Version
215
- version: '1.0'
216
- type: :runtime
217
- prerelease: false
218
- version_requirements: !ruby/object:Gem::Requirement
219
- requirements:
220
- - - "~>"
221
- - !ruby/object:Gem::Version
222
- version: '1.0'
223
- - !ruby/object:Gem::Dependency
224
- name: hooks
225
- requirement: !ruby/object:Gem::Requirement
226
- requirements:
227
- - - "~>"
201
+ version: '4.4'
202
+ - - "<"
228
203
  - !ruby/object:Gem::Version
229
- version: '0.4'
204
+ version: '4.5'
230
205
  type: :runtime
231
206
  prerelease: false
232
207
  version_requirements: !ruby/object:Gem::Requirement
233
208
  requirements:
234
209
  - - "~>"
235
210
  - !ruby/object:Gem::Version
236
- version: '0.4'
211
+ version: '4.4'
212
+ - - "<"
213
+ - !ruby/object:Gem::Version
214
+ version: '4.5'
237
215
  - !ruby/object:Gem::Dependency
238
- name: bloomfilter-rb
216
+ name: selenium-webdriver
239
217
  requirement: !ruby/object:Gem::Requirement
240
218
  requirements:
241
219
  - - "~>"
242
220
  - !ruby/object:Gem::Version
243
- version: '2.1'
221
+ version: '3.4'
244
222
  type: :runtime
245
223
  prerelease: false
246
224
  version_requirements: !ruby/object:Gem::Requirement
247
225
  requirements:
248
226
  - - "~>"
249
227
  - !ruby/object:Gem::Version
250
- version: '2.1'
228
+ version: '3.4'
251
229
  - !ruby/object:Gem::Dependency
252
- name: oj
230
+ name: thor
253
231
  requirement: !ruby/object:Gem::Requirement
254
232
  requirements:
255
233
  - - "~>"
256
234
  - !ruby/object:Gem::Version
257
- version: '2.12'
235
+ version: '1.0'
258
236
  type: :runtime
259
237
  prerelease: false
260
238
  version_requirements: !ruby/object:Gem::Requirement
261
239
  requirements:
262
240
  - - "~>"
263
241
  - !ruby/object:Gem::Version
264
- version: '2.12'
242
+ version: '1.0'
265
243
  - !ruby/object:Gem::Dependency
266
- name: fast_trie
244
+ name: cuprite
267
245
  requirement: !ruby/object:Gem::Requirement
268
246
  requirements:
269
247
  - - "~>"
270
248
  - !ruby/object:Gem::Version
271
- version: '0.5'
272
- type: :runtime
249
+ version: '0.13'
250
+ type: :development
273
251
  prerelease: false
274
252
  version_requirements: !ruby/object:Gem::Requirement
275
253
  requirements:
276
254
  - - "~>"
277
255
  - !ruby/object:Gem::Version
278
- version: '0.5'
256
+ version: '0.13'
279
257
  - !ruby/object:Gem::Dependency
280
- name: parallel
258
+ name: factory_bot
281
259
  requirement: !ruby/object:Gem::Requirement
282
260
  requirements:
283
261
  - - "~>"
284
262
  - !ruby/object:Gem::Version
285
- version: '1.11'
286
- type: :runtime
263
+ version: '6.0'
264
+ type: :development
287
265
  prerelease: false
288
266
  version_requirements: !ruby/object:Gem::Requirement
289
267
  requirements:
290
268
  - - "~>"
291
269
  - !ruby/object:Gem::Version
292
- version: '1.11'
270
+ version: '6.0'
293
271
  - !ruby/object:Gem::Dependency
294
- name: normalize_url
272
+ name: faker
295
273
  requirement: !ruby/object:Gem::Requirement
296
274
  requirements:
297
275
  - - "~>"
298
276
  - !ruby/object:Gem::Version
299
- version: 0.0.6
300
- type: :runtime
277
+ version: '1.7'
278
+ type: :development
301
279
  prerelease: false
302
280
  version_requirements: !ruby/object:Gem::Requirement
303
281
  requirements:
304
282
  - - "~>"
305
283
  - !ruby/object:Gem::Version
306
- version: 0.0.6
284
+ version: '1.7'
307
285
  - !ruby/object:Gem::Dependency
308
286
  name: pry
309
287
  requirement: !ruby/object:Gem::Requirement
@@ -338,42 +316,42 @@ dependencies:
338
316
  requirements:
339
317
  - - "~>"
340
318
  - !ruby/object:Gem::Version
341
- version: '3.3'
319
+ version: '3.4'
342
320
  type: :development
343
321
  prerelease: false
344
322
  version_requirements: !ruby/object:Gem::Requirement
345
323
  requirements:
346
324
  - - "~>"
347
325
  - !ruby/object:Gem::Version
348
- version: '3.3'
326
+ version: '3.4'
349
327
  - !ruby/object:Gem::Dependency
350
- name: sinatra
328
+ name: rubocop
351
329
  requirement: !ruby/object:Gem::Requirement
352
330
  requirements:
353
331
  - - "~>"
354
332
  - !ruby/object:Gem::Version
355
- version: '1.4'
333
+ version: '0.49'
356
334
  type: :development
357
335
  prerelease: false
358
336
  version_requirements: !ruby/object:Gem::Requirement
359
337
  requirements:
360
338
  - - "~>"
361
339
  - !ruby/object:Gem::Version
362
- version: '1.4'
340
+ version: '0.49'
363
341
  - !ruby/object:Gem::Dependency
364
- name: rubocop
342
+ name: sinatra
365
343
  requirement: !ruby/object:Gem::Requirement
366
344
  requirements:
367
345
  - - "~>"
368
346
  - !ruby/object:Gem::Version
369
- version: '0.34'
347
+ version: '1.4'
370
348
  type: :development
371
349
  prerelease: false
372
350
  version_requirements: !ruby/object:Gem::Requirement
373
351
  requirements:
374
352
  - - "~>"
375
353
  - !ruby/object:Gem::Version
376
- version: '0.34'
354
+ version: '1.4'
377
355
  - !ruby/object:Gem::Dependency
378
356
  name: yard
379
357
  requirement: !ruby/object:Gem::Requirement
@@ -388,310 +366,205 @@ dependencies:
388
366
  - - "~>"
389
367
  - !ruby/object:Gem::Version
390
368
  version: '0.9'
391
- - !ruby/object:Gem::Dependency
392
- name: bourbon
393
- requirement: !ruby/object:Gem::Requirement
394
- requirements:
395
- - - "~>"
396
- - !ruby/object:Gem::Version
397
- version: '4.3'
398
- type: :development
399
- prerelease: false
400
- version_requirements: !ruby/object:Gem::Requirement
401
- requirements:
402
- - - "~>"
403
- - !ruby/object:Gem::Version
404
- version: '4.3'
405
- - !ruby/object:Gem::Dependency
406
- name: neat
407
- requirement: !ruby/object:Gem::Requirement
408
- requirements:
409
- - - "~>"
410
- - !ruby/object:Gem::Version
411
- version: '1.8'
412
- type: :development
413
- prerelease: false
414
- version_requirements: !ruby/object:Gem::Requirement
415
- requirements:
416
- - - "~>"
417
- - !ruby/object:Gem::Version
418
- version: '1.8'
419
- - !ruby/object:Gem::Dependency
420
- name: faker
421
- requirement: !ruby/object:Gem::Requirement
422
- requirements:
423
- - - "~>"
424
- - !ruby/object:Gem::Version
425
- version: '1.7'
426
- type: :development
427
- prerelease: false
428
- version_requirements: !ruby/object:Gem::Requirement
429
- requirements:
430
- - - "~>"
431
- - !ruby/object:Gem::Version
432
- version: '1.7'
433
- description: Versatile web crawling with Ruby
434
- email: bauerdominic@gmail.com
369
+ description: Web crawling framework based on ActiveJob
370
+ email: mail@bauerd.com
435
371
  executables:
436
372
  - wayfarer
437
373
  extensions: []
438
374
  extra_rdoc_files: []
439
375
  files:
376
+ - ".github/workflows/ci.yaml"
440
377
  - ".gitignore"
441
378
  - ".rbenv-gemsets"
442
379
  - ".rspec"
443
380
  - ".rubocop.yml"
444
381
  - ".ruby-version"
445
- - ".travis.yml"
446
382
  - ".yardopts"
447
- - Changelog.md
383
+ - Dockerfile
448
384
  - Gemfile
385
+ - Gemfile.lock
449
386
  - LICENSE
450
- - README.md
387
+ - RELEASING.md
451
388
  - Rakefile
452
- - benchmark/frontiers.rb
453
389
  - bin/wayfarer
454
- - docs/.gitignore
455
- - docs/_config.yml
456
- - docs/_includes/base.html
457
- - docs/_includes/head.html
458
- - docs/_includes/navigation.html
459
- - docs/_layouts/default.html
460
- - docs/_sass/base.scss
461
- - docs/_sass/variables.scss
462
- - docs/_sass/vendor/bourbon/_bourbon-deprecate.scss
463
- - docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss
464
- - docs/_sass/vendor/bourbon/_bourbon.scss
465
- - docs/_sass/vendor/bourbon/addons/_border-color.scss
466
- - docs/_sass/vendor/bourbon/addons/_border-radius.scss
467
- - docs/_sass/vendor/bourbon/addons/_border-style.scss
468
- - docs/_sass/vendor/bourbon/addons/_border-width.scss
469
- - docs/_sass/vendor/bourbon/addons/_buttons.scss
470
- - docs/_sass/vendor/bourbon/addons/_clearfix.scss
471
- - docs/_sass/vendor/bourbon/addons/_ellipsis.scss
472
- - docs/_sass/vendor/bourbon/addons/_font-stacks.scss
473
- - docs/_sass/vendor/bourbon/addons/_hide-text.scss
474
- - docs/_sass/vendor/bourbon/addons/_margin.scss
475
- - docs/_sass/vendor/bourbon/addons/_padding.scss
476
- - docs/_sass/vendor/bourbon/addons/_position.scss
477
- - docs/_sass/vendor/bourbon/addons/_prefixer.scss
478
- - docs/_sass/vendor/bourbon/addons/_retina-image.scss
479
- - docs/_sass/vendor/bourbon/addons/_size.scss
480
- - docs/_sass/vendor/bourbon/addons/_text-inputs.scss
481
- - docs/_sass/vendor/bourbon/addons/_timing-functions.scss
482
- - docs/_sass/vendor/bourbon/addons/_triangle.scss
483
- - docs/_sass/vendor/bourbon/addons/_word-wrap.scss
484
- - docs/_sass/vendor/bourbon/css3/_animation.scss
485
- - docs/_sass/vendor/bourbon/css3/_appearance.scss
486
- - docs/_sass/vendor/bourbon/css3/_backface-visibility.scss
487
- - docs/_sass/vendor/bourbon/css3/_background-image.scss
488
- - docs/_sass/vendor/bourbon/css3/_background.scss
489
- - docs/_sass/vendor/bourbon/css3/_border-image.scss
490
- - docs/_sass/vendor/bourbon/css3/_calc.scss
491
- - docs/_sass/vendor/bourbon/css3/_columns.scss
492
- - docs/_sass/vendor/bourbon/css3/_filter.scss
493
- - docs/_sass/vendor/bourbon/css3/_flex-box.scss
494
- - docs/_sass/vendor/bourbon/css3/_font-face.scss
495
- - docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss
496
- - docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss
497
- - docs/_sass/vendor/bourbon/css3/_hyphens.scss
498
- - docs/_sass/vendor/bourbon/css3/_image-rendering.scss
499
- - docs/_sass/vendor/bourbon/css3/_keyframes.scss
500
- - docs/_sass/vendor/bourbon/css3/_linear-gradient.scss
501
- - docs/_sass/vendor/bourbon/css3/_perspective.scss
502
- - docs/_sass/vendor/bourbon/css3/_placeholder.scss
503
- - docs/_sass/vendor/bourbon/css3/_radial-gradient.scss
504
- - docs/_sass/vendor/bourbon/css3/_selection.scss
505
- - docs/_sass/vendor/bourbon/css3/_text-decoration.scss
506
- - docs/_sass/vendor/bourbon/css3/_transform.scss
507
- - docs/_sass/vendor/bourbon/css3/_transition.scss
508
- - docs/_sass/vendor/bourbon/css3/_user-select.scss
509
- - docs/_sass/vendor/bourbon/functions/_assign-inputs.scss
510
- - docs/_sass/vendor/bourbon/functions/_contains-falsy.scss
511
- - docs/_sass/vendor/bourbon/functions/_contains.scss
512
- - docs/_sass/vendor/bourbon/functions/_is-length.scss
513
- - docs/_sass/vendor/bourbon/functions/_is-light.scss
514
- - docs/_sass/vendor/bourbon/functions/_is-number.scss
515
- - docs/_sass/vendor/bourbon/functions/_is-size.scss
516
- - docs/_sass/vendor/bourbon/functions/_modular-scale.scss
517
- - docs/_sass/vendor/bourbon/functions/_px-to-em.scss
518
- - docs/_sass/vendor/bourbon/functions/_px-to-rem.scss
519
- - docs/_sass/vendor/bourbon/functions/_shade.scss
520
- - docs/_sass/vendor/bourbon/functions/_strip-units.scss
521
- - docs/_sass/vendor/bourbon/functions/_tint.scss
522
- - docs/_sass/vendor/bourbon/functions/_transition-property-name.scss
523
- - docs/_sass/vendor/bourbon/functions/_unpack.scss
524
- - docs/_sass/vendor/bourbon/helpers/_convert-units.scss
525
- - docs/_sass/vendor/bourbon/helpers/_directional-values.scss
526
- - docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss
527
- - docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss
528
- - docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss
529
- - docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss
530
- - docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss
531
- - docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss
532
- - docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss
533
- - docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss
534
- - docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss
535
- - docs/_sass/vendor/bourbon/helpers/_render-gradients.scss
536
- - docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss
537
- - docs/_sass/vendor/bourbon/helpers/_str-to-num.scss
538
- - docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss
539
- - docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss
540
- - docs/_sass/vendor/bourbon/settings/_prefixer.scss
541
- - docs/_sass/vendor/bourbon/settings/_px-to-em.scss
542
- - docs/_sass/vendor/neat/_neat-helpers.scss
543
- - docs/_sass/vendor/neat/_neat.scss
544
- - docs/_sass/vendor/neat/functions/_new-breakpoint.scss
545
- - docs/_sass/vendor/neat/functions/_private.scss
546
- - docs/_sass/vendor/neat/grid/_box-sizing.scss
547
- - docs/_sass/vendor/neat/grid/_direction-context.scss
548
- - docs/_sass/vendor/neat/grid/_display-context.scss
549
- - docs/_sass/vendor/neat/grid/_fill-parent.scss
550
- - docs/_sass/vendor/neat/grid/_media.scss
551
- - docs/_sass/vendor/neat/grid/_omega.scss
552
- - docs/_sass/vendor/neat/grid/_outer-container.scss
553
- - docs/_sass/vendor/neat/grid/_pad.scss
554
- - docs/_sass/vendor/neat/grid/_private.scss
555
- - docs/_sass/vendor/neat/grid/_row.scss
556
- - docs/_sass/vendor/neat/grid/_shift.scss
557
- - docs/_sass/vendor/neat/grid/_span-columns.scss
558
- - docs/_sass/vendor/neat/grid/_to-deprecate.scss
559
- - docs/_sass/vendor/neat/grid/_visual-grid.scss
560
- - docs/_sass/vendor/neat/mixins/_clearfix.scss
561
- - docs/_sass/vendor/neat/settings/_disable-warnings.scss
562
- - docs/_sass/vendor/neat/settings/_grid.scss
563
- - docs/_sass/vendor/neat/settings/_visual-grid.scss
564
- - docs/_sass/vendor/normalize-3.0.2.scss
565
- - docs/_sass/vendor/pygments.scss
566
- - docs/automating_browsers/capybara.md
567
- - docs/css/screen.scss
390
+ - docker-compose.yml
391
+ - docs/cookbook/batch_routing.md
392
+ - docs/cookbook/consent_screen.md
393
+ - docs/cookbook/executing_javascript.md
394
+ - docs/cookbook/querying_html.md
395
+ - docs/cookbook/screenshots.md
396
+ - docs/cookbook/user_agent.md
397
+ - docs/guides/browser_automation/capybara.md
398
+ - docs/guides/browser_automation/custom_adapters.md
399
+ - docs/guides/browser_automation/ferrum.md
400
+ - docs/guides/browser_automation/selenium.md
568
401
  - docs/guides/callbacks.md
569
- - docs/guides/cli.md
570
402
  - docs/guides/configuration.md
403
+ - docs/guides/debugging.md
571
404
  - docs/guides/error_handling.md
572
- - docs/guides/frontiers.md
573
- - docs/guides/halting.md
574
- - docs/guides/job_queues.md
575
- - docs/guides/locals.md
576
- - docs/guides/logging.md
577
- - docs/guides/page_objects.md
578
- - docs/guides/peeking.md
579
- - docs/guides/selenium_capybara.md
580
- - docs/guides/tutorial.md
405
+ - docs/guides/jobs.md
406
+ - docs/guides/navigation.md
407
+ - docs/guides/networking.md
408
+ - docs/guides/pages.md
409
+ - docs/guides/performance.md
410
+ - docs/guides/reliability.md
411
+ - docs/guides/routing/steering.md
412
+ - docs/guides/tasks.md
581
413
  - docs/index.md
582
- - docs/js/navigation.js
583
- - docs/misc/contributing.md
584
- - docs/misc/testing.md
585
- - docs/recipes/authentication.md
586
- - docs/recipes/csv.md
587
- - docs/recipes/javascript.md
588
- - docs/recipes/multiple_uris.md
589
- - docs/recipes/screenshots.md
590
- - docs/routing/custom_rules.md
591
- - docs/routing/filetypes_rules.md
592
- - docs/routing/host_rules.md
593
- - docs/routing/path_rules.md
594
- - docs/routing/protocol_rules.md
595
- - docs/routing/query_rules.md
596
- - docs/routing/routes.md
597
- - docs/routing/uri_rules.md
598
- - examples/collect_github_issues.rb
599
- - examples/find_foobar_on_wikipedia.rb
414
+ - docs/reference/api/base.md
415
+ - docs/reference/api/route.md
416
+ - docs/reference/cli.md
417
+ - docs/reference/configuration_keys.md
418
+ - docs/reference/environment_variables.md
600
419
  - lib/wayfarer.rb
601
- - lib/wayfarer/configuration.rb
602
- - lib/wayfarer/crawl.rb
603
- - lib/wayfarer/crawl_observer.rb
604
- - lib/wayfarer/dispatcher.rb
605
- - lib/wayfarer/finders.rb
606
- - lib/wayfarer/frontiers/frontier.rb
607
- - lib/wayfarer/frontiers/memory_bloomfilter.rb
608
- - lib/wayfarer/frontiers/memory_frontier.rb
609
- - lib/wayfarer/frontiers/memory_trie_frontier.rb
610
- - lib/wayfarer/frontiers/normalize_uris.rb
611
- - lib/wayfarer/frontiers/redis_bloomfilter.rb
612
- - lib/wayfarer/frontiers/redis_frontier.rb
613
- - lib/wayfarer/http_adapters/adapter_pool.rb
614
- - lib/wayfarer/http_adapters/net_http_adapter.rb
615
- - lib/wayfarer/http_adapters/selenium_adapter.rb
616
- - lib/wayfarer/job.rb
617
- - lib/wayfarer/locals.rb
420
+ - lib/wayfarer/base.rb
421
+ - lib/wayfarer/callbacks.rb
422
+ - lib/wayfarer/cli/base.rb
423
+ - lib/wayfarer/cli/generate.rb
424
+ - lib/wayfarer/cli/job.rb
425
+ - lib/wayfarer/cli/route.rb
426
+ - lib/wayfarer/cli/route_printer.rb
427
+ - lib/wayfarer/cli/runner.rb
428
+ - lib/wayfarer/cli/templates/Gemfile.tt
429
+ - lib/wayfarer/cli/templates/job.rb.tt
430
+ - lib/wayfarer/config/capybara.rb
431
+ - lib/wayfarer/config/ferrum.rb
432
+ - lib/wayfarer/config/networking.rb
433
+ - lib/wayfarer/config/redis.rb
434
+ - lib/wayfarer/config/root.rb
435
+ - lib/wayfarer/config/selenium.rb
436
+ - lib/wayfarer/config/strconv.rb
437
+ - lib/wayfarer/config/struct.rb
438
+ - lib/wayfarer/gc.rb
439
+ - lib/wayfarer/middleware/chain.rb
440
+ - lib/wayfarer/middleware/dedup.rb
441
+ - lib/wayfarer/middleware/fetch.rb
442
+ - lib/wayfarer/middleware/normalize.rb
443
+ - lib/wayfarer/middleware/router.rb
444
+ - lib/wayfarer/middleware/stage.rb
445
+ - lib/wayfarer/middleware/worker.rb
446
+ - lib/wayfarer/networking/capybara.rb
447
+ - lib/wayfarer/networking/context.rb
448
+ - lib/wayfarer/networking/ferrum.rb
449
+ - lib/wayfarer/networking/http.rb
450
+ - lib/wayfarer/networking/pool.rb
451
+ - lib/wayfarer/networking/result.rb
452
+ - lib/wayfarer/networking/selenium.rb
453
+ - lib/wayfarer/networking/strategy.rb
618
454
  - lib/wayfarer/page.rb
619
- - lib/wayfarer/parsers/json_parser.rb
620
- - lib/wayfarer/parsers/xml_parser.rb
621
- - lib/wayfarer/processor.rb
622
- - lib/wayfarer/routing/custom_rule.rb
623
- - lib/wayfarer/routing/filetypes_rule.rb
624
- - lib/wayfarer/routing/host_rule.rb
625
- - lib/wayfarer/routing/path_rule.rb
626
- - lib/wayfarer/routing/protocol_rule.rb
627
- - lib/wayfarer/routing/query_rule.rb
455
+ - lib/wayfarer/parsing/json.rb
456
+ - lib/wayfarer/parsing/xml.rb
457
+ - lib/wayfarer/redis/.#barrier.rb
458
+ - lib/wayfarer/redis/barrier.rb
459
+ - lib/wayfarer/redis/connection.rb
460
+ - lib/wayfarer/redis/counter.rb
461
+ - lib/wayfarer/redis/pool.rb
462
+ - lib/wayfarer/redis/version.rb
463
+ - lib/wayfarer/routing/dsl.rb
464
+ - lib/wayfarer/routing/matchers/custom.rb
465
+ - lib/wayfarer/routing/matchers/host.rb
466
+ - lib/wayfarer/routing/matchers/path.rb
467
+ - lib/wayfarer/routing/matchers/query.rb
468
+ - lib/wayfarer/routing/matchers/scheme.rb
469
+ - lib/wayfarer/routing/matchers/suffix.rb
470
+ - lib/wayfarer/routing/matchers/url.rb
471
+ - lib/wayfarer/routing/path_finder.rb
472
+ - lib/wayfarer/routing/result.rb
473
+ - lib/wayfarer/routing/root_route.rb
474
+ - lib/wayfarer/routing/route.rb
628
475
  - lib/wayfarer/routing/router.rb
629
- - lib/wayfarer/routing/rule.rb
630
- - lib/wayfarer/routing/uri_rule.rb
631
- - spec/configuration_spec.rb
632
- - spec/crawl_spec.rb
633
- - spec/finders_spec.rb
634
- - spec/frontiers/memory_bloomfilter_spec.rb
635
- - spec/frontiers/memory_frontier_spec.rb
636
- - spec/frontiers/memory_trie_frontier_spec.rb
637
- - spec/frontiers/normalize_uris_spec.rb
638
- - spec/frontiers/redis_bloomfilter_spec.rb
639
- - spec/frontiers/redis_frontier_spec.rb
640
- - spec/http_adapters/adapter_pool_spec.rb
641
- - spec/http_adapters/net_http_adapter_spec.rb
642
- - spec/http_adapters/selenium_adapter_spec.rb
643
- - spec/integration/callbacks_spec.rb
644
- - spec/integration/locals_spec.rb
645
- - spec/integration/peeking_spec.rb
646
- - spec/job_spec.rb
476
+ - lib/wayfarer/routing/target_route.rb
477
+ - lib/wayfarer/serializer.rb
478
+ - lib/wayfarer/stringify.rb
479
+ - lib/wayfarer/task.rb
480
+ - mkdocs.yml
481
+ - requirements.txt
482
+ - spec/base_spec.rb
483
+ - spec/callbacks_spec.rb
484
+ - spec/cli/generate_spec.rb
485
+ - spec/cli/job_spec.rb
486
+ - spec/cli/version_spec.rb
487
+ - spec/config/capybara_spec.rb
488
+ - spec/config/ferrum_spec.rb
489
+ - spec/config/networking_spec.rb
490
+ - spec/config/redis_spec.rb
491
+ - spec/config/root_spec.rb
492
+ - spec/config/selenium_spec.rb
493
+ - spec/config/strconv_spec.rb
494
+ - spec/config/struct_spec.rb
495
+ - spec/factories/middleware.rb
496
+ - spec/factories/page.rb
497
+ - spec/factories/task.rb
498
+ - spec/fixtures/dummy_job.rb
499
+ - spec/gc_spec.rb
500
+ - spec/middleware/chain_spec.rb
501
+ - spec/middleware/dedup_spec.rb
502
+ - spec/middleware/fetch_spec.rb
503
+ - spec/middleware/normalize_spec.rb
504
+ - spec/middleware/router_spec.rb
505
+ - spec/middleware/stage_spec.rb
506
+ - spec/middleware/worker_spec.rb
507
+ - spec/networking/capybara_spec.rb
508
+ - spec/networking/context_spec.rb
509
+ - spec/networking/ferrum_spec.rb
510
+ - spec/networking/http_spec.rb
511
+ - spec/networking/pool_spec.rb
512
+ - spec/networking/selenium_spec.rb
513
+ - spec/networking/strategy.rb
647
514
  - spec/page_spec.rb
648
- - spec/parsers/json_parser_spec.rb
649
- - spec/parsers/xml_parser_spec.rb
650
- - spec/processor_spec.rb
651
- - spec/routing/custom_rule_spec.rb
652
- - spec/routing/filetypes_rule_spec.rb
653
- - spec/routing/host_rule_spec.rb
654
- - spec/routing/path_rule_spec.rb
655
- - spec/routing/protocol_rule_spec.rb
656
- - spec/routing/query_rule_spec.rb
515
+ - spec/parsing/json_spec.rb
516
+ - spec/parsing/xml_spec.rb
517
+ - spec/redis/barrier_spec.rb
518
+ - spec/redis/counter_spec.rb
519
+ - spec/redis/pool_spec.rb
520
+ - spec/redis/version_spec.rb
521
+ - spec/routing/dsl_spec.rb
522
+ - spec/routing/integration_spec.rb
523
+ - spec/routing/matchers/custom_spec.rb
524
+ - spec/routing/matchers/host_spec.rb
525
+ - spec/routing/matchers/path_spec.rb
526
+ - spec/routing/matchers/query_spec.rb
527
+ - spec/routing/matchers/scheme_spec.rb
528
+ - spec/routing/matchers/suffix_spec.rb
529
+ - spec/routing/matchers/uri_spec.rb
530
+ - spec/routing/path_finder_spec.rb
531
+ - spec/routing/root_route_spec.rb
532
+ - spec/routing/route_spec.rb
657
533
  - spec/routing/router_spec.rb
658
- - spec/routing/rule_spec.rb
659
- - spec/routing/uri_rule_spec.rb
660
- - spec/shared/frontier.rb
661
534
  - spec/spec_helpers.rb
535
+ - spec/stringify_spec.rb
536
+ - spec/support/static/finders.html
537
+ - spec/support/static/graph/details/a.html
538
+ - spec/support/static/graph/details/b.html
539
+ - spec/support/static/graph/index.html
540
+ - spec/support/static/json/dummy.json
541
+ - spec/support/static/links/links.html
542
+ - spec/support/static/xml/dummy.xml
543
+ - spec/support/test_app.rb
544
+ - spec/task_spec.rb
662
545
  - spec/wayfarer_spec.rb
663
- - support/static/finders.html
664
- - support/static/graph/details/a.html
665
- - support/static/graph/details/b.html
666
- - support/static/graph/index.html
667
- - support/static/json/dummy.json
668
- - support/static/links/links.html
669
- - support/static/xml/dummy.xml
670
- - support/test_app.rb
671
- - wayfarer-jruby.gemspec
672
546
  - wayfarer.gemspec
673
547
  homepage: http://github.com/bauerd/wayfarer
674
548
  licenses:
675
549
  - MIT
676
550
  metadata: {}
677
- post_install_message:
551
+ post_install_message:
678
552
  rdoc_options: []
679
553
  require_paths:
680
554
  - lib
681
555
  required_ruby_version: !ruby/object:Gem::Requirement
682
556
  requirements:
683
- - - ">="
557
+ - - "~>"
684
558
  - !ruby/object:Gem::Version
685
- version: '0'
559
+ version: '2.7'
686
560
  required_rubygems_version: !ruby/object:Gem::Requirement
687
561
  requirements:
688
562
  - - ">="
689
563
  - !ruby/object:Gem::Version
690
564
  version: '0'
691
565
  requirements: []
692
- rubyforge_project:
693
- rubygems_version: 2.5.1
694
- signing_key:
566
+ rubygems_version: 3.1.6
567
+ signing_key:
695
568
  specification_version: 4
696
569
  summary: Versatile web crawling with Ruby
697
570
  test_files: []