fetch_util 0.5.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +49 -0
  3. data/README.md +3 -1
  4. data/Rakefile +0 -2
  5. data/lib/fetch_util/assets/extract.js +1 -1
  6. data/lib/fetch_util/browser/interaction_helpers/consent_helpers.rb +12 -17
  7. data/lib/fetch_util/browser/interaction_helpers/timing_helpers.rb +42 -15
  8. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script.rb +48 -0
  9. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script.rb +80 -0
  10. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_helpers_script.rb +53 -0
  11. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_state_script.rb +118 -0
  12. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_stabilization.rb +54 -0
  13. data/lib/fetch_util/browser/site_stabilization/azure_devops_pr_state.rb +27 -0
  14. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_product_state.rb +55 -0
  15. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_request_state.rb +100 -0
  16. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_state.rb +59 -0
  17. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_modules.rb +26 -0
  18. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_state.rb +93 -0
  19. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resources.rb +73 -0
  20. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_thread_state.rb +92 -0
  21. data/lib/fetch_util/browser/site_stabilization/bitbucket_cloud_threads.rb +71 -0
  22. data/lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb +9 -10
  23. data/lib/fetch_util/browser/site_stabilization/facebook_stabilization.rb +56 -0
  24. data/lib/fetch_util/browser/site_stabilization/gerrit_change_product_state_script.rb +97 -0
  25. data/lib/fetch_util/browser/site_stabilization/gerrit_change_request_state_script.rb +75 -0
  26. data/lib/fetch_util/browser/site_stabilization/gerrit_change_stabilization.rb +37 -0
  27. data/lib/fetch_util/browser/site_stabilization/gerrit_change_state.rb +39 -0
  28. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_fetch_state_script.rb +31 -0
  29. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_product_state_script.rb +76 -0
  30. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_request_state_script.rb +111 -0
  31. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_route_state_script.rb +71 -0
  32. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_stabilization.rb +37 -0
  33. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_state.rb +42 -0
  34. data/lib/fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script.rb +49 -0
  35. data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_product_state.rb +88 -0
  36. data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_state.rb +109 -0
  37. data/lib/fetch_util/browser/site_stabilization/gitea_family_pull_resources.rb +30 -0
  38. data/lib/fetch_util/browser/site_stabilization/gitea_family_stabilization.rb +60 -0
  39. data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_product_state.rb +74 -0
  40. data/lib/fetch_util/browser/site_stabilization/gitea_family_thread_timeline_state.rb +69 -0
  41. data/lib/fetch_util/browser/site_stabilization/gitea_family_threads.rb +28 -0
  42. data/lib/fetch_util/browser/site_stabilization/gitea_family_visibility_script.rb +28 -0
  43. data/lib/fetch_util/browser/site_stabilization/github_pull_resource_stabilization.rb +31 -0
  44. data/lib/fetch_util/browser/site_stabilization/github_pull_resources.rb +105 -0
  45. data/lib/fetch_util/browser/site_stabilization/github_threads.rb +82 -0
  46. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state.rb +17 -0
  47. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state_script.rb +98 -0
  48. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_visibility_script.rb +43 -0
  49. data/lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resources.rb +28 -0
  50. data/lib/fetch_util/browser/site_stabilization/gitlab_repo.rb +3 -11
  51. data/lib/fetch_util/browser/site_stabilization/gitlab_stabilization.rb +33 -0
  52. data/lib/fetch_util/browser/site_stabilization/gitlab_threads.rb +92 -0
  53. data/lib/fetch_util/browser/site_stabilization/social_platforms.rb +19 -38
  54. data/lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb +3 -3
  55. data/lib/fetch_util/browser/site_stabilization.rb +89 -0
  56. data/lib/fetch_util/browser/stabilization/anubis.rb +4 -5
  57. data/lib/fetch_util/browser/stabilization/page_flow.rb +71 -27
  58. data/lib/fetch_util/browser/stabilization/spa_hydration.rb +7 -5
  59. data/lib/fetch_util/browser.rb +88 -17
  60. data/lib/fetch_util/cli.rb +69 -23
  61. data/lib/fetch_util/extractor.rb +8 -2
  62. data/lib/fetch_util/fetcher.rb +144 -43
  63. data/lib/fetch_util/http_redirect_client.rb +161 -0
  64. data/lib/fetch_util/parallel_fetcher.rb +125 -28
  65. data/lib/fetch_util/raw_docs_fallback.rb +158 -21
  66. data/lib/fetch_util/regulatory/cache_store.rb +47 -14
  67. data/lib/fetch_util/regulatory/fetch_records.rb +10 -7
  68. data/lib/fetch_util/regulatory/headers.rb +8 -1
  69. data/lib/fetch_util/regulatory/http_client.rb +2 -97
  70. data/lib/fetch_util/regulatory/orchestration.rb +10 -6
  71. data/lib/fetch_util/regulatory/page.rb +8 -11
  72. data/lib/fetch_util/regulatory/robots.rb +2 -10
  73. data/lib/fetch_util/regulatory/source_selection.rb +13 -3
  74. data/lib/fetch_util/regulatory/tdm_page.rb +9 -4
  75. data/lib/fetch_util/regulatory/tdm_policy.rb +16 -9
  76. data/lib/fetch_util/regulatory/tdm_rep.rb +4 -3
  77. data/lib/fetch_util/regulatory/tdm_support.rb +10 -6
  78. data/lib/fetch_util/regulatory.rb +5 -1
  79. data/lib/fetch_util/request_log.rb +24 -3
  80. data/lib/fetch_util/result.rb +37 -89
  81. data/lib/fetch_util/search_transport.rb +102 -35
  82. data/lib/fetch_util/searcher.rb +15 -13
  83. data/lib/fetch_util/version.rb +1 -1
  84. data/lib/fetch_util.rb +4 -1
  85. metadata +66 -2
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fetch_util
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.2
4
+ version: 0.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - hmdne
@@ -37,6 +37,26 @@ dependencies:
37
37
  - - "~>"
38
38
  - !ruby/object:Gem::Version
39
39
  version: '1.19'
40
+ - !ruby/object:Gem::Dependency
41
+ name: public_suffix
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: '5.0'
47
+ - - "<"
48
+ - !ruby/object:Gem::Version
49
+ version: '8.0'
50
+ type: :runtime
51
+ prerelease: false
52
+ version_requirements: !ruby/object:Gem::Requirement
53
+ requirements:
54
+ - - ">="
55
+ - !ruby/object:Gem::Version
56
+ version: '5.0'
57
+ - - "<"
58
+ - !ruby/object:Gem::Version
59
+ version: '8.0'
40
60
  - !ruby/object:Gem::Dependency
41
61
  name: thor
42
62
  requirement: !ruby/object:Gem::Requirement
@@ -81,8 +101,51 @@ files:
81
101
  - lib/fetch_util/browser/navigation/headers_and_readiness.rb
82
102
  - lib/fetch_util/browser/navigation/navigator_patch.rb
83
103
  - lib/fetch_util/browser/site_stabilization.rb
104
+ - lib/fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script.rb
105
+ - lib/fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script.rb
106
+ - lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_helpers_script.rb
107
+ - lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_state_script.rb
108
+ - lib/fetch_util/browser/site_stabilization/azure_devops_pr_stabilization.rb
109
+ - lib/fetch_util/browser/site_stabilization/azure_devops_pr_state.rb
110
+ - lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_product_state.rb
111
+ - lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_request_state.rb
112
+ - lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_state.rb
113
+ - lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_modules.rb
114
+ - lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_state.rb
115
+ - lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resources.rb
116
+ - lib/fetch_util/browser/site_stabilization/bitbucket_cloud_thread_state.rb
117
+ - lib/fetch_util/browser/site_stabilization/bitbucket_cloud_threads.rb
84
118
  - lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb
119
+ - lib/fetch_util/browser/site_stabilization/facebook_stabilization.rb
120
+ - lib/fetch_util/browser/site_stabilization/gerrit_change_product_state_script.rb
121
+ - lib/fetch_util/browser/site_stabilization/gerrit_change_request_state_script.rb
122
+ - lib/fetch_util/browser/site_stabilization/gerrit_change_stabilization.rb
123
+ - lib/fetch_util/browser/site_stabilization/gerrit_change_state.rb
124
+ - lib/fetch_util/browser/site_stabilization/gerrit_file_resource_fetch_state_script.rb
125
+ - lib/fetch_util/browser/site_stabilization/gerrit_file_resource_product_state_script.rb
126
+ - lib/fetch_util/browser/site_stabilization/gerrit_file_resource_request_state_script.rb
127
+ - lib/fetch_util/browser/site_stabilization/gerrit_file_resource_route_state_script.rb
128
+ - lib/fetch_util/browser/site_stabilization/gerrit_file_resource_stabilization.rb
129
+ - lib/fetch_util/browser/site_stabilization/gerrit_file_resource_state.rb
130
+ - lib/fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script.rb
131
+ - lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_product_state.rb
132
+ - lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_state.rb
133
+ - lib/fetch_util/browser/site_stabilization/gitea_family_pull_resources.rb
134
+ - lib/fetch_util/browser/site_stabilization/gitea_family_stabilization.rb
135
+ - lib/fetch_util/browser/site_stabilization/gitea_family_thread_product_state.rb
136
+ - lib/fetch_util/browser/site_stabilization/gitea_family_thread_timeline_state.rb
137
+ - lib/fetch_util/browser/site_stabilization/gitea_family_threads.rb
138
+ - lib/fetch_util/browser/site_stabilization/gitea_family_visibility_script.rb
139
+ - lib/fetch_util/browser/site_stabilization/github_pull_resource_stabilization.rb
140
+ - lib/fetch_util/browser/site_stabilization/github_pull_resources.rb
141
+ - lib/fetch_util/browser/site_stabilization/github_threads.rb
142
+ - lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state.rb
143
+ - lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state_script.rb
144
+ - lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_visibility_script.rb
145
+ - lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resources.rb
85
146
  - lib/fetch_util/browser/site_stabilization/gitlab_repo.rb
147
+ - lib/fetch_util/browser/site_stabilization/gitlab_stabilization.rb
148
+ - lib/fetch_util/browser/site_stabilization/gitlab_threads.rb
86
149
  - lib/fetch_util/browser/site_stabilization/social_platforms.rb
87
150
  - lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb
88
151
  - lib/fetch_util/browser/stabilization.rb
@@ -92,6 +155,7 @@ files:
92
155
  - lib/fetch_util/cli.rb
93
156
  - lib/fetch_util/extractor.rb
94
157
  - lib/fetch_util/fetcher.rb
158
+ - lib/fetch_util/http_redirect_client.rb
95
159
  - lib/fetch_util/parallel_fetcher.rb
96
160
  - lib/fetch_util/raw_docs_fallback.rb
97
161
  - lib/fetch_util/regulatory.rb
@@ -143,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
143
207
  - !ruby/object:Gem::Version
144
208
  version: '0'
145
209
  requirements: []
146
- rubygems_version: 4.0.15
210
+ rubygems_version: 4.0.19
147
211
  specification_version: 4
148
212
  summary: AI for fetching in Ruby
149
213
  test_files: []