fetch_util 0.7.0 → 0.8.0

data/lib/fetch_util/browser/site_stabilization.rb

@@ -3,104 +3,17 @@
 module FetchUtil
   class Browser
     module SiteStabilization
-      autoload :AzureDevopsPrProductStateScript,
-               "fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script"
-      autoload :AzureDevopsPrCommitDetailsScript,
-               "fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script"
-      autoload :AzureDevopsPrRequestHelpersScript,
-               "fetch_util/browser/site_stabilization/azure_devops_pr_request_helpers_script"
-      autoload :AzureDevopsPrRequestStateScript,
-               "fetch_util/browser/site_stabilization/azure_devops_pr_request_state_script"
-      autoload :AzureDevopsPrState, "fetch_util/browser/site_stabilization/azure_devops_pr_state"
-      autoload :AzureDevopsPrStabilization,
-               "fetch_util/browser/site_stabilization/azure_devops_pr_stabilization"
-      autoload :BitbucketCloudThreadState,
-               "fetch_util/browser/site_stabilization/bitbucket_cloud_thread_state"
-      autoload :BitbucketCloudThreads, "fetch_util/browser/site_stabilization/bitbucket_cloud_threads"
-      autoload :BitbucketCloudPullResourceModules,
-               "fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_modules"
-      autoload :CommunityAndMarketplace, "fetch_util/browser/site_stabilization/community_and_marketplace"
-      autoload :FacebookStabilization, "fetch_util/browser/site_stabilization/facebook_stabilization"
-      autoload :GerritChangeProductStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_change_product_state_script"
-      autoload :GerritChangeRequestStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_change_request_state_script"
-      autoload :GerritChangeState, "fetch_util/browser/site_stabilization/gerrit_change_state"
-      autoload :GerritChangeStabilization,
-               "fetch_util/browser/site_stabilization/gerrit_change_stabilization"
-      autoload :GerritFileResourceProductStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_product_state_script"
-      autoload :GerritFileResourceRouteStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_route_state_script"
-      autoload :GerritFileResourceFetchStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_fetch_state_script"
-      autoload :GerritFileResourceValidationStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script"
-      autoload :GerritFileResourceRequestStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_request_state_script"
-      autoload :GerritFileResourceState,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_state"
-      autoload :GerritFileResourceStabilization,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_stabilization"
-      autoload :GithubPullResources, "fetch_util/browser/site_stabilization/github_pull_resources"
-      autoload :GithubPullResourceStabilization,
-               "fetch_util/browser/site_stabilization/github_pull_resource_stabilization"
-      autoload :GithubThreads, "fetch_util/browser/site_stabilization/github_threads"
-      autoload :GiteaFamilyThreadProductState,
-               "fetch_util/browser/site_stabilization/gitea_family_thread_product_state"
-      autoload :GiteaFamilyThreadTimelineState,
-               "fetch_util/browser/site_stabilization/gitea_family_thread_timeline_state"
-      autoload :GiteaFamilyStabilization,
-               "fetch_util/browser/site_stabilization/gitea_family_stabilization"
-      autoload :GiteaFamilyVisibilityScript,
-               "fetch_util/browser/site_stabilization/gitea_family_visibility_script"
-      autoload :GiteaFamilyThreads, "fetch_util/browser/site_stabilization/gitea_family_threads"
-      autoload :GiteaFamilyPullResourceProductState,
-               "fetch_util/browser/site_stabilization/gitea_family_pull_resource_product_state"
-      autoload :GiteaFamilyPullResourceState,
-               "fetch_util/browser/site_stabilization/gitea_family_pull_resource_state"
-      autoload :GiteaFamilyPullResources, "fetch_util/browser/site_stabilization/gitea_family_pull_resources"
-      autoload :GitlabRepo, "fetch_util/browser/site_stabilization/gitlab_repo"
-      autoload :GitlabMergeRequestResourceVisibilityScript,
-               "fetch_util/browser/site_stabilization/gitlab_merge_request_resource_visibility_script"
-      autoload :GitlabMergeRequestResourceStateScript,
-               "fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state_script"
-      autoload :GitlabMergeRequestResourceState,
-               "fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state"
-      autoload :GitlabMergeRequestResources, "fetch_util/browser/site_stabilization/gitlab_merge_request_resources"
-      autoload :GitlabStabilization, "fetch_util/browser/site_stabilization/gitlab_stabilization"
-      autoload :GitlabThreads, "fetch_util/browser/site_stabilization/gitlab_threads"
-      autoload :SocialPlatforms, "fetch_util/browser/site_stabilization/social_platforms"
-      autoload :TravelAndLodging, "fetch_util/browser/site_stabilization/travel_and_lodging"
+      autoload :CommunityStabilization, "fetch_util/browser/site_stabilization/communities"
+      autoload :ForgeStabilization, "fetch_util/browser/site_stabilization/forges"
+      autoload :MarketplaceStabilization, "fetch_util/browser/site_stabilization/marketplaces"
+      autoload :SocialStabilization, "fetch_util/browser/site_stabilization/social"
+      autoload :TravelStabilization, "fetch_util/browser/site_stabilization/travel"
 
-      include AzureDevopsPrState
-      include AzureDevopsPrStabilization
-      include BitbucketCloudThreadState
-      include BitbucketCloudThreads
-      include BitbucketCloudPullResourceModules
-      include CommunityAndMarketplace
-      include FacebookStabilization
-      include GerritChangeState
-      include GerritChangeStabilization
-      include GerritFileResourceState
-      include GerritFileResourceStabilization
-      include GithubPullResources
-      include GithubPullResourceStabilization
-      include GithubThreads
-      include GiteaFamilyVisibilityScript
-      include GiteaFamilyThreadProductState
-      include GiteaFamilyThreadTimelineState
-      include GiteaFamilyStabilization
-      include GiteaFamilyThreads
-      include GiteaFamilyPullResourceProductState
-      include GiteaFamilyPullResourceState
-      include GiteaFamilyPullResources
-      include GitlabRepo
-      include GitlabStabilization
-      include GitlabMergeRequestResources
-      include GitlabThreads
-      include SocialPlatforms
-      include TravelAndLodging
+      include CommunityStabilization
+      include ForgeStabilization
+      include MarketplaceStabilization
+      include SocialStabilization
+      include TravelStabilization
     end
   end
 end

data/lib/fetch_util/browser/stabilization/page_flow.rb

@@ -6,10 +6,10 @@ module FetchUtil
     module Stabilization
       module PageFlow
         PAGE_FLOW_STABILIZATION_PROFILES = [
-          SiteStabilization::CommunityAndMarketplace::COMMUNITY_MARKETPLACE_STABILIZATION_PROFILES[:stabilize_reddit],
-          SiteStabilization::SocialPlatforms::SOCIAL_PLATFORM_STABILIZATION_PROFILES[:stabilize_instagram],
-          SiteStabilization::SocialPlatforms::SOCIAL_PLATFORM_STABILIZATION_PROFILES[:stabilize_facebook],
-          SiteStabilization::CommunityAndMarketplace::COMMUNITY_MARKETPLACE_STABILIZATION_PROFILES[:stabilize_ebay_search],
+          SiteStabilization::RedditStabilization::REDDIT_STABILIZATION_PROFILE,
+          SiteStabilization::InstagramStabilization::INSTAGRAM_STABILIZATION_PROFILE,
+          SiteStabilization::FacebookStabilization::FACEBOOK_STABILIZATION_PROFILE,
+          SiteStabilization::EbayStabilization::EBAY_STABILIZATION_PROFILE,
           SiteStabilization::GithubThreads::GITHUB_THREAD_STABILIZATION_PROFILE,
           SiteStabilization::GithubPullResources::GITHUB_PULL_RESOURCE_STABILIZATION_PROFILE,
           SiteStabilization::GitlabThreads::GITLAB_THREAD_STABILIZATION_PROFILE,
@@ -21,7 +21,7 @@ module FetchUtil
           SiteStabilization::AzureDevopsPrStabilization::AZURE_DEVOPS_PR_STABILIZATION_PROFILE,
           SiteStabilization::GerritFileResourceStabilization::GERRIT_FILE_RESOURCE_STABILIZATION_PROFILE,
           SiteStabilization::GerritChangeStabilization::GERRIT_CHANGE_STABILIZATION_PROFILE,
-          SiteStabilization::TravelAndLodging::TRAVEL_LODGING_STABILIZATION_PROFILES[:stabilize_lodging_detail],
+          SiteStabilization::LodgingStabilization::LODGING_STABILIZATION_PROFILE,
           { host: "t.me", path_query: ->(uri) { uri.path.match?(%r{\A/s/[^/]+/\d+/?\z}) },
             strategy: :wait_for_telegram_message, notes: "Wait for the requested public Telegram preview message.",
             tests: "spec/fetch_util/browser_stabilization_spec.rb" },
@@ -95,6 +95,8 @@ module FetchUtil
 
         def matching_stabilization_profile(url, profiles)
           uri = URI.parse(url)
+          return nil unless uri.is_a?(URI::HTTP) && uri.host
+
           host = FetchUtil.strip_www_host(url)
           profiles.find { |profile| profile_match?(profile, uri, host) }
         rescue URI::InvalidURIError

data/lib/fetch_util/browser.rb

@@ -233,6 +233,9 @@ module FetchUtil
           raise unless page_loaded_enough?(page)
         end
         stabilize_page(page, loaded_page_url(page, url))
+        if page.current_url == "about:blank" && url != "about:blank"
+          raise Ferrum::Error, "Request to #{url} finished on about:blank without a public page"
+        end
         page
       rescue Ferrum::PendingConnectionsError, Ferrum::TimeoutError, Ferrum::Error => e
         close_page(page)

data/lib/fetch_util/raw_docs_fallback.rb

@@ -300,7 +300,7 @@ module FetchUtil
     def markdown_from_root(root, title)
       sections = []
       blocks = root.css(BLOCK_SELECTOR).to_a
-      blocks.unshift(root) if %w[ul ol].include?(root.name)
+      blocks.unshift(root) if BLOCK_ELEMENTS.include?(root.name)
       blocks.each do |node|
         next if node.ancestors.any? { |ancestor| %w[ul ol].include?(ancestor.name) }
 
@@ -320,6 +320,10 @@ module FetchUtil
           sections << [fence, code, fence].join("\n")
         when "ul", "ol"
           sections << markdown_list(node)
+        when "li"
+          content = node.dup
+          content.css("ul, ol").remove
+          sections << "- #{clean_text(content.text)}"
         when "tr"
           cells = node.css("th, td").map { |cell| clean_text(cell.text) }.reject(&:empty?)
           sections << "- #{cells.join(": ")}" unless cells.empty?

data/lib/fetch_util/regulatory/cache_store.rb

@@ -3,12 +3,14 @@
 module FetchUtil
   class Regulatory
     module CacheStore
+      include CacheValidation
+
       private
 
-      def cache_fetch(key)
+      def cache_fetch(key, shape: nil)
         path = cache_file_path(key)
         cached = read_cache(path)
-        return cached if cached
+        return cached if cached && (!shape || cache_payload_matches?(cached, shape))
 
         payload, cacheable = yield
         begin
@@ -20,7 +22,7 @@ module FetchUtil
       end
 
       def fetch_record(key, uri, fallback: nil, require_success: true)
-        cache_fetch(key) do
+        cache_fetch(key, shape: fallback) do
           response, cacheable = record_response(uri, require_success: require_success)
           payload = response ? yield(response.body, response) : fallback
           [payload, cacheable]

data/lib/fetch_util/regulatory/cache_validation.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module CacheValidation
+      private
+
+      def cache_payload_matches?(payload, shape)
+        case shape
+        when Hash
+          payload.is_a?(Hash) && shape.all? do |key, value_shape|
+            payload.key?(key) && cache_payload_matches?(payload[key], value_shape)
+          end
+        when Array
+          payload.is_a?(Array)
+        when nil
+          payload.nil? || payload.is_a?(String)
+        else
+          payload.is_a?(shape.class)
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/tdm_policy.rb

@@ -20,7 +20,7 @@ module FetchUtil
       end
 
       def tdm_policy_record(url, target_origin:)
-        cache_fetch("tdmpolicy:#{origin_key(target_origin)}:#{url}") do
+        cache_fetch("tdmpolicy:#{origin_key(target_origin)}:#{url}", shape: { "signals" => [] }) do
           response, cacheable = safe_get(url)
           signals = []
           if response&.status&.between?(200, 299) && json_like_response?(response.headers, response.body)

data/lib/fetch_util/regulatory.rb

@@ -43,6 +43,7 @@ module FetchUtil
     autoload :SourceSelection, "fetch_util/regulatory/source_selection"
     autoload :Signals, "fetch_util/regulatory/signals"
     autoload :FetchRecords, "fetch_util/regulatory/fetch_records"
+    autoload :CacheValidation, "fetch_util/regulatory/cache_validation"
     autoload :CacheStore, "fetch_util/regulatory/cache_store"
     autoload :Robots, "fetch_util/regulatory/robots"
     autoload :RobotGlobs, "fetch_util/regulatory/robot_globs"

data/lib/fetch_util/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FetchUtil
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fetch_util
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - hmdne
@@ -102,53 +102,60 @@ files:
 - lib/fetch_util/browser/navigation/navigator_patch.rb
 - lib/fetch_util/browser/navigation/shadow_root_patch.rb
 - lib/fetch_util/browser/site_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script.rb
-- lib/fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script.rb
-- lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_helpers_script.rb
-- lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_state_script.rb
-- lib/fetch_util/browser/site_stabilization/azure_devops_pr_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/azure_devops_pr_state.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_product_state.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_request_state.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_state.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_modules.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_state.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resources.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_thread_state.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_threads.rb
-- lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb
-- lib/fetch_util/browser/site_stabilization/facebook_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_change_product_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_change_request_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_change_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_change_state.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_fetch_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_product_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_request_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_route_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_state.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_product_state.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_state.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_pull_resources.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_thread_product_state.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_thread_timeline_state.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_threads.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_visibility_script.rb
-- lib/fetch_util/browser/site_stabilization/github_pull_resource_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/github_pull_resources.rb
-- lib/fetch_util/browser/site_stabilization/github_threads.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_visibility_script.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resources.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_repo.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_threads.rb
-- lib/fetch_util/browser/site_stabilization/social_platforms.rb
-- lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb
+- lib/fetch_util/browser/site_stabilization/communities.rb
+- lib/fetch_util/browser/site_stabilization/communities/reddit.rb
+- lib/fetch_util/browser/site_stabilization/forges.rb
+- lib/fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/commit_details_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/product_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/request_helpers_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/request_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/stabilization.rb
+- lib/fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/state.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/pulls/diff/product_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/pulls/diff/request_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/pulls/diff/state.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/pulls/resource_modules.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/pulls/resource_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/pulls/resources.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/threads/index.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/threads/state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/changes/product_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/changes/request_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/changes/stabilization.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/changes/state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/fetch_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/product_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/request_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/route_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/stabilization.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/validation_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/pulls/product_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/pulls/resources.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/pulls/state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/stabilization.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/threads/index.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/threads/product_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/threads/timeline_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/visibility_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/github/pull_resource_stabilization.rb
+- lib/fetch_util/browser/site_stabilization/forges/github/pull_resources.rb
+- lib/fetch_util/browser/site_stabilization/forges/github/threads.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/merge_requests/resource_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/merge_requests/resource_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/merge_requests/resource_visibility_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/merge_requests/resources.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/repository.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/stabilization.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/threads.rb
+- lib/fetch_util/browser/site_stabilization/forges/observation_state.rb
+- lib/fetch_util/browser/site_stabilization/marketplaces.rb
+- lib/fetch_util/browser/site_stabilization/marketplaces/ebay.rb
+- lib/fetch_util/browser/site_stabilization/social.rb
+- lib/fetch_util/browser/site_stabilization/social/facebook.rb
+- lib/fetch_util/browser/site_stabilization/social/instagram.rb
+- lib/fetch_util/browser/site_stabilization/travel.rb
+- lib/fetch_util/browser/site_stabilization/travel/lodging.rb
 - lib/fetch_util/browser/stabilization.rb
 - lib/fetch_util/browser/stabilization/anubis.rb
 - lib/fetch_util/browser/stabilization/page_flow.rb
@@ -161,6 +168,7 @@ files:
 - lib/fetch_util/raw_docs_fallback.rb
 - lib/fetch_util/regulatory.rb
 - lib/fetch_util/regulatory/cache_store.rb
+- lib/fetch_util/regulatory/cache_validation.rb
 - lib/fetch_util/regulatory/directives.rb
 - lib/fetch_util/regulatory/fetch_records.rb
 - lib/fetch_util/regulatory/headers.rb

data/lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb

@@ -1,117 +0,0 @@
-# frozen_string_literal: true
-
-module FetchUtil
-  class Browser
-    module SiteStabilization
-      module CommunityAndMarketplace
-        COMMUNITY_MARKETPLACE_STABILIZATION_PROFILES = {
-          stabilize_reddit: {
-            host: "reddit.com",
-            strategy: :stabilize_reddit,
-            notes: "Use fast Reddit cookie dismissal/content readiness instead of full idle waits.",
-            tests: "spec/fetch_util/browser_stabilization_spec.rb"
-          },
-          stabilize_ebay_search: {
-            host: "ebay.com",
-            path_query: ->(uri) { uri.path.include?("/sch/") || uri.query.to_s.include?("_nkw=") },
-            strategy: :stabilize_ebay_search,
-            notes: "Wait for search result items or short-lived eBay browser checks.",
-            tests: "spec/fetch_util/browser_stabilization_spec.rb"
-          }
-        }.freeze
-
-        private
-
-        def stabilize_reddit(page, deadline: stabilization_deadline)
-          ready = retry_until_timeout(capped_timeout(3.0, deadline: deadline), interval: 0.1, deadline: deadline) do
-            dismiss_reddit_cookie_dialog(page)
-            reddit_content_ready?(page)
-          end
-
-          settle_after_stabilization(0.25, deadline: deadline)
-          dismiss_reddit_cookie_dialog(page) if stabilization_time_remaining?(deadline)
-          ready
-        end
-
-        def stabilize_ebay_search(page, deadline: stabilization_deadline)
-          accepted_cookies = false
-          cookie_config = consent_config(
-            accept_labels: [
-              "accept all",
-              "accept all cookies",
-              "accept cookies",
-              "allow all",
-              "allow cookies",
-              "agree to cookies",
-              "continue with cookies"
-            ]
-          )
-
-          retry_until_timeout(capped_timeout(6.0, deadline: deadline), interval: 0.15, deadline: deadline) do
-            accepted_cookies ||= click_visible_button_by_text(
-              page,
-              consent_accept_labels(cookie_config),
-              selectors: consent_button_selectors(cookie_config)
-            )
-
-            state = safe_evaluate(page, <<~JS, default: { "itemCount" => 0, "challengeVisible" => false })
-              (() => {
-                const bodyText = document.body ? document.body.innerText : '';
-                return {
-                  itemCount: document.querySelectorAll('li.s-item a[href*="/itm/"], ul.srp-results a[href*="/itm/"]').length,
-                  challengeVisible: /checking your browser before you access ebay|your browser will redirect to your requested content shortly|pardon our interruption/i.test(bodyText)
-                };
-              })()
-            JS
-
-            state["itemCount"].to_i >= 4 || (state["challengeVisible"] ? 0.35 : false)
-          end
-
-          settle_after_stabilization(0.25, deadline: deadline) if accepted_cookies
-        end
-
-        def reddit_content_ready?(page)
-          safe_evaluate(page, <<~JS, default: false)
-            !!document.querySelector('shreddit-post, faceplate-screen-reader-content, shreddit-comment, [data-testid="comment"]')
-          JS
-        end
-
-        def dismiss_reddit_cookie_dialog(page)
-          removed = dismiss_overlay_dialog(
-            page,
-            close_selectors: [],
-            dialog_selectors: [
-              '[data-testid="onboarding-modal"]',
-              '[data-testid="gdpr-modal"]',
-              '[aria-modal="true"]',
-              '[role="dialog"]',
-              "shreddit-experience-tree"
-            ],
-            dialog_pattern: "before you continue to reddit|let us know your cookie preferences"
-          )
-          return true if removed
-
-          safe_evaluate(page, <<~JS)
-            (() => {
-              #{js_dom_helpers}
-              let removed = false;
-              document.querySelectorAll('section, div, aside, form, footer, shreddit-experience-tree').forEach((node) => {
-                const text = (node.innerText || node.textContent || '').replace(/\s+/g, ' ').trim();
-                if (/before you continue to reddit|let us know your cookie preferences/i.test(text) && text.length < 2000) {
-                  node.remove();
-                  removed = true;
-                }
-              });
-
-              if (removed) {
-                restoreScroll();
-              }
-
-              return removed;
-            })()
-          JS
-        end
-      end
-    end
-  end
-end

data/lib/fetch_util/browser/site_stabilization/gitea_family_stabilization.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-module FetchUtil
-  class Browser
-    module SiteStabilization
-      module GiteaFamilyStabilization
-        private
-
-        def stabilize_gitea_family_state(page, state_script, deadline: stabilization_deadline)
-          last_signature = nil
-          stable_observations = 0
-          incomplete_signature = nil
-          incomplete_observations = 0
-          terminal_incomplete = false
-          observed = retry_until_timeout(
-            capped_timeout(6.0, deadline: deadline),
-            interval: 0.1,
-            deadline: deadline
-          ) do
-            state = safe_evaluate(page, state_script, default: nil)
-            return false if state.is_a?(Hash) && state["product"] == false
-            next false unless state.is_a?(Hash) && state["product"]
-
-            unless state["ready"]
-              if state["loading"]
-                incomplete_signature = nil
-                incomplete_observations = 0
-              elsif state["signature"] == incomplete_signature
-                incomplete_observations += 1
-              else
-                incomplete_signature = state["signature"]
-                incomplete_observations = 1
-              end
-              if incomplete_observations >= 10
-                terminal_incomplete = true
-                next true
-              end
-              next false
-            end
-
-            incomplete_signature = nil
-            incomplete_observations = 0
-            if state["signature"] == last_signature
-              stable_observations += 1
-            else
-              last_signature = state["signature"]
-              stable_observations = 1
-            end
-            stable_observations >= 3
-          end
-
-          return false if terminal_incomplete
-
-          settle_after_stabilization(0.5, deadline: deadline) if observed
-          !!observed
-        end
-      end
-    end
-  end
-end