fetch_util 0.3.0 → 0.3.1

data/lib/fetch_util/browser/site_stabilization/gitlab_repo.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      module GitlabRepo
+        private
+
+        def gitlab_repo_url?(url)
+          uri = URI.parse(url)
+          host = uri.host.to_s.downcase
+          (host == "gitlab.com" || host.end_with?(".gitlab.com")) && uri.path.split("/").reject(&:empty?).length == 2
+        rescue URI::InvalidURIError
+          false
+        end
+
+        def stabilize_gitlab_repo(page)
+          retry_until_timeout(capped_timeout(8.0), interval: 0.2) do
+            safe_evaluate(page, <<~JS, default: 0).to_i >= 300
+              (() => {
+                const readme = document.querySelector('[data-testid="blob-viewer-content"] .blob-viewer[data-path="README.md"] .file-content.md, .blob-viewer[data-path="README.md"] .file-content.md');
+                return readme ? (readme.innerText || readme.textContent || '').replace(/\s+/g, ' ').trim().length : 0;
+              })()
+            JS
+          end
+
+          settle_after_stabilization(0.25)
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization.rb

@@ -4,9 +4,11 @@ module FetchUtil
   class Browser
     module SiteStabilization
       autoload :CommunityAndMarketplace, "fetch_util/browser/site_stabilization/community_and_marketplace"
+      autoload :GitlabRepo, "fetch_util/browser/site_stabilization/gitlab_repo"
       autoload :SocialPlatforms, "fetch_util/browser/site_stabilization/social_platforms"
 
       include CommunityAndMarketplace
+      include GitlabRepo
       include SocialPlatforms
     end
   end

data/lib/fetch_util/browser/stabilization/page_flow.rb

@@ -11,6 +11,7 @@ module FetchUtil
           return stabilize_instagram(page) if instagram_url?(url)
           return stabilize_facebook(page) if facebook_url?(url)
           return stabilize_ebay_search(page) if ebay_search_url?(url)
+          return stabilize_gitlab_repo(page) if gitlab_repo_url?(url)
 
           reached_idle = !@wait_for_idle || wait_for_idle_or_content(page)
           preserve_consent = preserve_consent_wall?(page, url)

data/lib/fetch_util/cli.rb

@@ -17,6 +17,7 @@ module FetchUtil
       content_type
       suspect
       warnings
+      error_message
     ].freeze
 
     class_option :log_path, type: :string, desc: "Append-only request log path"