fetch_util 0.7.0 → 0.8.0

data/lib/fetch_util/browser/site_stabilization/communities/reddit.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      module RedditStabilization
+        REDDIT_STABILIZATION_PROFILE = {
+          host: "reddit.com",
+          strategy: :stabilize_reddit,
+          notes: "Use fast Reddit cookie dismissal/content readiness instead of full idle waits.",
+          tests: "spec/fetch_util/browser_stabilization_spec.rb"
+        }.freeze
+
+        private
+
+        def stabilize_reddit(page, deadline: stabilization_deadline)
+          ready = retry_until_timeout(capped_timeout(3.0, deadline: deadline), interval: 0.1, deadline: deadline) do
+            dismiss_reddit_cookie_dialog(page)
+            reddit_content_ready?(page)
+          end
+
+          settle_after_stabilization(0.25, deadline: deadline)
+          dismiss_reddit_cookie_dialog(page) if stabilization_time_remaining?(deadline)
+          ready
+        end
+
+        def reddit_content_ready?(page)
+          safe_evaluate(page, <<~JS, default: false)
+            !!document.querySelector('shreddit-post, faceplate-screen-reader-content, shreddit-comment, [data-testid="comment"]')
+          JS
+        end
+
+        def dismiss_reddit_cookie_dialog(page)
+          removed = dismiss_overlay_dialog(
+            page,
+            close_selectors: [],
+            dialog_selectors: [
+              '[data-testid="onboarding-modal"]',
+              '[data-testid="gdpr-modal"]',
+              '[aria-modal="true"]',
+              '[role="dialog"]',
+              "shreddit-experience-tree"
+            ],
+            dialog_pattern: "before you continue to reddit|let us know your cookie preferences"
+          )
+          return true if removed
+
+          safe_evaluate(page, <<~JS)
+            (() => {
+              #{js_dom_helpers}
+              let removed = false;
+              document.querySelectorAll('section, div, aside, form, footer, shreddit-experience-tree').forEach((node) => {
+                const text = (node.innerText || node.textContent || '').replace(/\s+/g, ' ').trim();
+                if (/before you continue to reddit|let us know your cookie preferences/i.test(text) && text.length < 2000) {
+                  node.remove();
+                  removed = true;
+                }
+              });
+
+              if (removed) {
+                restoreScroll();
+              }
+
+              return removed;
+            })()
+          JS
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization/communities.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      autoload :RedditStabilization, "fetch_util/browser/site_stabilization/communities/reddit"
+
+      module CommunityStabilization
+        include RedditStabilization
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization/{bitbucket_cloud_pull_resource_modules.rb → forges/bitbucket/pulls/resource_modules.rb}

@@ -4,15 +4,15 @@ module FetchUtil
   class Browser
     module SiteStabilization
       autoload :BitbucketCloudPullDiffProductState,
-               "fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_product_state"
+               "fetch_util/browser/site_stabilization/forges/bitbucket/pulls/diff/product_state"
       autoload :BitbucketCloudPullDiffRequestState,
-               "fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_request_state"
+               "fetch_util/browser/site_stabilization/forges/bitbucket/pulls/diff/request_state"
       autoload :BitbucketCloudPullDiffState,
-               "fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_state"
+               "fetch_util/browser/site_stabilization/forges/bitbucket/pulls/diff/state"
       autoload :BitbucketCloudPullResourceState,
-               "fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_state"
+               "fetch_util/browser/site_stabilization/forges/bitbucket/pulls/resource_state"
       autoload :BitbucketCloudPullResources,
-               "fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resources"
+               "fetch_util/browser/site_stabilization/forges/bitbucket/pulls/resources"
 
       module BitbucketCloudPullResourceModules
         include BitbucketCloudPullDiffProductState

data/lib/fetch_util/browser/site_stabilization/{bitbucket_cloud_pull_resources.rb → forges/bitbucket/pulls/resources.rb}

@@ -16,11 +16,7 @@ module FetchUtil
         private
 
         def stabilize_bitbucket_cloud_pull_resource(page, deadline: stabilization_deadline)
-          last_signature = nil
-          stable_observations = 0
-          incomplete_signature = nil
-          incomplete_observations = 0
-          terminal_incomplete = false
+          observation = ForgeObservationState.new(reset_stability_when_incomplete: true)
           observed = retry_until_timeout(
             capped_timeout(6.0, deadline: deadline),
             interval: 0.1,
@@ -30,37 +26,10 @@ module FetchUtil
             return false if state.is_a?(Hash) && state["product"] == false
             next false unless state.is_a?(Hash) && state["product"]
 
-            unless state["ready"]
-              last_signature = nil
-              stable_observations = 0
-              if state["loading"]
-                incomplete_signature = nil
-                incomplete_observations = 0
-              elsif state["signature"] == incomplete_signature
-                incomplete_observations += 1
-              else
-                incomplete_signature = state["signature"]
-                incomplete_observations = 1
-              end
-              if incomplete_observations >= 10
-                terminal_incomplete = true
-                next true
-              end
-              next false
-            end
-
-            incomplete_signature = nil
-            incomplete_observations = 0
-            if state["signature"] == last_signature
-              stable_observations += 1
-            else
-              last_signature = state["signature"]
-              stable_observations = 1
-            end
-            stable_observations >= 3
+            observation.observe(state)
           end
 
-          return false if terminal_incomplete
+          return false if observation.terminal_incomplete?
 
           fail_bitbucket_cloud_pull_diff_preparation(page) unless observed
 

data/lib/fetch_util/browser/site_stabilization/{bitbucket_cloud_threads.rb → forges/bitbucket/threads/index.rb}

@@ -16,11 +16,7 @@ module FetchUtil
         private
 
         def stabilize_bitbucket_cloud_thread(page, deadline: stabilization_deadline)
-          last_signature = nil
-          stable_observations = 0
-          incomplete_signature = nil
-          incomplete_observations = 0
-          terminal_incomplete = false
+          observation = ForgeObservationState.new(reset_stability_when_incomplete: true)
           observed = retry_until_timeout(
             capped_timeout(6.0, deadline: deadline),
             interval: 0.1,
@@ -30,37 +26,10 @@ module FetchUtil
             return false if state.is_a?(Hash) && state["product"] == false
             next false unless state.is_a?(Hash) && state["product"]
 
-            unless state["ready"]
-              last_signature = nil
-              stable_observations = 0
-              if state["loading"]
-                incomplete_signature = nil
-                incomplete_observations = 0
-              elsif state["signature"] == incomplete_signature
-                incomplete_observations += 1
-              else
-                incomplete_signature = state["signature"]
-                incomplete_observations = 1
-              end
-              if incomplete_observations >= 10
-                terminal_incomplete = true
-                next true
-              end
-              next false
-            end
-
-            incomplete_signature = nil
-            incomplete_observations = 0
-            if state["signature"] == last_signature
-              stable_observations += 1
-            else
-              last_signature = state["signature"]
-              stable_observations = 1
-            end
-            stable_observations >= 3
+            observation.observe(state)
           end
 
-          return false if terminal_incomplete
+          return false if observation.terminal_incomplete?
 
           settle_after_stabilization(0.5, deadline: deadline) if observed
           !!observed

data/lib/fetch_util/browser/site_stabilization/forges/gitea/stabilization.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      module GiteaFamilyStabilization
+        private
+
+        def stabilize_gitea_family_state(page, state_script, deadline: stabilization_deadline)
+          observation = ForgeObservationState.new(reset_stability_when_incomplete: false)
+          observed = retry_until_timeout(
+            capped_timeout(6.0, deadline: deadline),
+            interval: 0.1,
+            deadline: deadline
+          ) do
+            state = safe_evaluate(page, state_script, default: nil)
+            return false if state.is_a?(Hash) && state["product"] == false
+            next false unless state.is_a?(Hash) && state["product"]
+
+            observation.observe(state)
+          end
+
+          return false if observation.terminal_incomplete?
+
+          settle_after_stabilization(0.5, deadline: deadline) if observed
+          !!observed
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization/forges/observation_state.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      class ForgeObservationState
+        REQUIRED_STABLE_OBSERVATIONS = 3
+        REQUIRED_INCOMPLETE_OBSERVATIONS = 10
+
+        attr_reader :terminal_incomplete
+        alias terminal_incomplete? terminal_incomplete
+
+        def initialize(reset_stability_when_incomplete:)
+          @reset_stability_when_incomplete = reset_stability_when_incomplete
+          @last_signature = nil
+          @stable_observations = 0
+          @incomplete_signature = nil
+          @incomplete_observations = 0
+          @terminal_incomplete = false
+        end
+
+        def observe(state)
+          return observe_incomplete(state) unless state["ready"]
+
+          @incomplete_signature = nil
+          @incomplete_observations = 0
+          if state["signature"] == @last_signature
+            @stable_observations += 1
+          else
+            @last_signature = state["signature"]
+            @stable_observations = 1
+          end
+          @stable_observations >= REQUIRED_STABLE_OBSERVATIONS
+        end
+
+        private
+
+        def observe_incomplete(state)
+          reset_stability if @reset_stability_when_incomplete
+          if state["loading"]
+            @incomplete_signature = nil
+            @incomplete_observations = 0
+          elsif state["signature"] == @incomplete_signature
+            @incomplete_observations += 1
+          else
+            @incomplete_signature = state["signature"]
+            @incomplete_observations = 1
+          end
+          @terminal_incomplete = @incomplete_observations >= REQUIRED_INCOMPLETE_OBSERVATIONS
+        end
+
+        def reset_stability
+          @last_signature = nil
+          @stable_observations = 0
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization/forges.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      autoload :AzureDevopsPrProductStateScript,
+               "fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/product_state_script"
+      autoload :AzureDevopsPrCommitDetailsScript,
+               "fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/commit_details_script"
+      autoload :AzureDevopsPrRequestHelpersScript,
+               "fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/request_helpers_script"
+      autoload :AzureDevopsPrRequestStateScript,
+               "fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/request_state_script"
+      autoload :AzureDevopsPrState,
+               "fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/state"
+      autoload :AzureDevopsPrStabilization,
+               "fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/stabilization"
+      autoload :BitbucketCloudThreadState,
+               "fetch_util/browser/site_stabilization/forges/bitbucket/threads/state"
+      autoload :BitbucketCloudThreads,
+               "fetch_util/browser/site_stabilization/forges/bitbucket/threads/index"
+      autoload :BitbucketCloudPullResourceModules,
+               "fetch_util/browser/site_stabilization/forges/bitbucket/pulls/resource_modules"
+      autoload :GerritChangeProductStateScript,
+               "fetch_util/browser/site_stabilization/forges/gerrit/changes/product_state_script"
+      autoload :GerritChangeRequestStateScript,
+               "fetch_util/browser/site_stabilization/forges/gerrit/changes/request_state_script"
+      autoload :GerritChangeState,
+               "fetch_util/browser/site_stabilization/forges/gerrit/changes/state"
+      autoload :GerritChangeStabilization,
+               "fetch_util/browser/site_stabilization/forges/gerrit/changes/stabilization"
+      autoload :GerritFileResourceProductStateScript,
+               "fetch_util/browser/site_stabilization/forges/gerrit/resources/product_state_script"
+      autoload :GerritFileResourceRouteStateScript,
+               "fetch_util/browser/site_stabilization/forges/gerrit/resources/route_state_script"
+      autoload :GerritFileResourceFetchStateScript,
+               "fetch_util/browser/site_stabilization/forges/gerrit/resources/fetch_state_script"
+      autoload :GerritFileResourceValidationStateScript,
+               "fetch_util/browser/site_stabilization/forges/gerrit/resources/validation_state_script"
+      autoload :GerritFileResourceRequestStateScript,
+               "fetch_util/browser/site_stabilization/forges/gerrit/resources/request_state_script"
+      autoload :GerritFileResourceState,
+               "fetch_util/browser/site_stabilization/forges/gerrit/resources/state"
+      autoload :GerritFileResourceStabilization,
+               "fetch_util/browser/site_stabilization/forges/gerrit/resources/stabilization"
+      autoload :GithubPullResources,
+               "fetch_util/browser/site_stabilization/forges/github/pull_resources"
+      autoload :GithubPullResourceStabilization,
+               "fetch_util/browser/site_stabilization/forges/github/pull_resource_stabilization"
+      autoload :GithubThreads, "fetch_util/browser/site_stabilization/forges/github/threads"
+      autoload :ForgeObservationState, "fetch_util/browser/site_stabilization/forges/observation_state"
+      autoload :GiteaFamilyThreadProductState,
+               "fetch_util/browser/site_stabilization/forges/gitea/threads/product_state"
+      autoload :GiteaFamilyThreadTimelineState,
+               "fetch_util/browser/site_stabilization/forges/gitea/threads/timeline_state"
+      autoload :GiteaFamilyStabilization,
+               "fetch_util/browser/site_stabilization/forges/gitea/stabilization"
+      autoload :GiteaFamilyVisibilityScript,
+               "fetch_util/browser/site_stabilization/forges/gitea/visibility_script"
+      autoload :GiteaFamilyThreads,
+               "fetch_util/browser/site_stabilization/forges/gitea/threads/index"
+      autoload :GiteaFamilyPullResourceProductState,
+               "fetch_util/browser/site_stabilization/forges/gitea/pulls/product_state"
+      autoload :GiteaFamilyPullResourceState,
+               "fetch_util/browser/site_stabilization/forges/gitea/pulls/state"
+      autoload :GiteaFamilyPullResources,
+               "fetch_util/browser/site_stabilization/forges/gitea/pulls/resources"
+      autoload :GitlabRepo, "fetch_util/browser/site_stabilization/forges/gitlab/repository"
+      autoload :GitlabMergeRequestResourceVisibilityScript,
+               "fetch_util/browser/site_stabilization/forges/gitlab/merge_requests/resource_visibility_script"
+      autoload :GitlabMergeRequestResourceStateScript,
+               "fetch_util/browser/site_stabilization/forges/gitlab/merge_requests/resource_state_script"
+      autoload :GitlabMergeRequestResourceState,
+               "fetch_util/browser/site_stabilization/forges/gitlab/merge_requests/resource_state"
+      autoload :GitlabMergeRequestResources,
+               "fetch_util/browser/site_stabilization/forges/gitlab/merge_requests/resources"
+      autoload :GitlabStabilization,
+               "fetch_util/browser/site_stabilization/forges/gitlab/stabilization"
+      autoload :GitlabThreads, "fetch_util/browser/site_stabilization/forges/gitlab/threads"
+
+      module ForgeStabilization
+        include AzureDevopsPrState
+        include AzureDevopsPrStabilization
+        include BitbucketCloudThreadState
+        include BitbucketCloudThreads
+        include BitbucketCloudPullResourceModules
+        include GerritChangeState
+        include GerritChangeStabilization
+        include GerritFileResourceState
+        include GerritFileResourceStabilization
+        include GithubPullResources
+        include GithubPullResourceStabilization
+        include GithubThreads
+        include GiteaFamilyVisibilityScript
+        include GiteaFamilyThreadProductState
+        include GiteaFamilyThreadTimelineState
+        include GiteaFamilyStabilization
+        include GiteaFamilyThreads
+        include GiteaFamilyPullResourceProductState
+        include GiteaFamilyPullResourceState
+        include GiteaFamilyPullResources
+        include GitlabRepo
+        include GitlabStabilization
+        include GitlabMergeRequestResources
+        include GitlabThreads
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization/marketplaces/ebay.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      module EbayStabilization
+        EBAY_STABILIZATION_PROFILE = {
+          host: "ebay.com",
+          path_query: ->(uri) { uri.path.include?("/sch/") || uri.query.to_s.include?("_nkw=") },
+          strategy: :stabilize_ebay_search,
+          notes: "Wait for search result items or short-lived eBay browser checks.",
+          tests: "spec/fetch_util/browser_stabilization_spec.rb"
+        }.freeze
+
+        private
+
+        def stabilize_ebay_search(page, deadline: stabilization_deadline)
+          accepted_cookies = false
+          cookie_config = consent_config(
+            accept_labels: [
+              "accept all",
+              "accept all cookies",
+              "accept cookies",
+              "allow all",
+              "allow cookies",
+              "agree to cookies",
+              "continue with cookies"
+            ]
+          )
+
+          retry_until_timeout(capped_timeout(6.0, deadline: deadline), interval: 0.15, deadline: deadline) do
+            accepted_cookies ||= click_visible_button_by_text(
+              page,
+              consent_accept_labels(cookie_config),
+              selectors: consent_button_selectors(cookie_config)
+            )
+
+            state = safe_evaluate(page, <<~JS, default: { "itemCount" => 0, "challengeVisible" => false })
+              (() => {
+                const bodyText = document.body ? document.body.innerText : '';
+                return {
+                  itemCount: document.querySelectorAll('li.s-item a[href*="/itm/"], ul.srp-results a[href*="/itm/"]').length,
+                  challengeVisible: /checking your browser before you access ebay|your browser will redirect to your requested content shortly|pardon our interruption/i.test(bodyText)
+                };
+              })()
+            JS
+
+            state["itemCount"].to_i >= 4 || (state["challengeVisible"] ? 0.35 : false)
+          end
+
+          settle_after_stabilization(0.25, deadline: deadline) if accepted_cookies
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization/marketplaces.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      autoload :EbayStabilization, "fetch_util/browser/site_stabilization/marketplaces/ebay"
+
+      module MarketplaceStabilization
+        include EbayStabilization
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization/{facebook_stabilization.rb → social/facebook.rb}

@@ -4,6 +4,13 @@ module FetchUtil
   class Browser
     module SiteStabilization
       module FacebookStabilization
+        FACEBOOK_STABILIZATION_PROFILE = {
+          host: "facebook.com",
+          strategy: :stabilize_facebook,
+          notes: "Decline/accept Facebook cookie dialogs, then dismiss login prompts.",
+          tests: "spec/fetch_util/browser_stabilization_spec.rb"
+        }.freeze
+
         private
 
         def stabilize_facebook(page, deadline: stabilization_deadline)

data/lib/fetch_util/browser/site_stabilization/{social_platforms.rb → social/instagram.rb}

@@ -3,14 +3,12 @@
 module FetchUtil
   class Browser
     module SiteStabilization
-      module SocialPlatforms
-        SOCIAL_PLATFORM_STABILIZATION_PROFILES = {
-          stabilize_instagram: { host: "instagram.com", strategy: :stabilize_instagram,
-                                 notes: "Accept Instagram cookie prompts and dismiss login modals before extraction.",
-                                 tests: "spec/fetch_util/browser_stabilization_spec.rb" },
-          stabilize_facebook: { host: "facebook.com", strategy: :stabilize_facebook,
-                                notes: "Decline/accept Facebook cookie dialogs, then dismiss login prompts.",
-                                tests: "spec/fetch_util/browser_stabilization_spec.rb" }
+      module InstagramStabilization
+        INSTAGRAM_STABILIZATION_PROFILE = {
+          host: "instagram.com",
+          strategy: :stabilize_instagram,
+          notes: "Accept Instagram cookie prompts and dismiss login modals before extraction.",
+          tests: "spec/fetch_util/browser_stabilization_spec.rb"
         }.freeze
 
         private

data/lib/fetch_util/browser/site_stabilization/social.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      autoload :FacebookStabilization, "fetch_util/browser/site_stabilization/social/facebook"
+      autoload :InstagramStabilization, "fetch_util/browser/site_stabilization/social/instagram"
+
+      module SocialStabilization
+        include FacebookStabilization
+        include InstagramStabilization
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization/{travel_and_lodging.rb → travel/lodging.rb}

@@ -3,15 +3,13 @@
 module FetchUtil
   class Browser
     module SiteStabilization
-      module TravelAndLodging
-        TRAVEL_LODGING_STABILIZATION_PROFILES = {
-          stabilize_lodging_detail: {
-            host: %w[airbnb.com booking.com],
-            path_query: ->(uri) { uri.path.match?(%r{/(?:rooms|hotel)/}) },
-            strategy: :stabilize_lodging_detail,
-            notes: "Wait for client-rendered lodging detail cues without bypassing access controls.",
-            tests: "spec/fetch_util/browser_stabilization_spec.rb"
-          }
+      module LodgingStabilization
+        LODGING_STABILIZATION_PROFILE = {
+          host: %w[airbnb.com booking.com],
+          path_query: ->(uri) { uri.path.match?(%r{/(?:rooms|hotel)/}) },
+          strategy: :stabilize_lodging_detail,
+          notes: "Wait for client-rendered lodging detail cues without bypassing access controls.",
+          tests: "spec/fetch_util/browser_stabilization_spec.rb"
         }.freeze
 
         private

data/lib/fetch_util/browser/site_stabilization/travel.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      autoload :LodgingStabilization, "fetch_util/browser/site_stabilization/travel/lodging"
+
+      module TravelStabilization
+        include LodgingStabilization
+      end
+    end
+  end
+end