fetch_util 0.6.2 → 0.8.0

data/lib/fetch_util/browser/site_stabilization/{travel_and_lodging.rb → travel/lodging.rb}

@@ -3,15 +3,13 @@
 module FetchUtil
   class Browser
     module SiteStabilization
-      module TravelAndLodging
-        TRAVEL_LODGING_STABILIZATION_PROFILES = {
-          stabilize_lodging_detail: {
-            host: %w[airbnb.com booking.com],
-            path_query: ->(uri) { uri.path.match?(%r{/(?:rooms|hotel)/}) },
-            strategy: :stabilize_lodging_detail,
-            notes: "Wait for client-rendered lodging detail cues without bypassing access controls.",
-            tests: "spec/fetch_util/browser_stabilization_spec.rb"
-          }
+      module LodgingStabilization
+        LODGING_STABILIZATION_PROFILE = {
+          host: %w[airbnb.com booking.com],
+          path_query: ->(uri) { uri.path.match?(%r{/(?:rooms|hotel)/}) },
+          strategy: :stabilize_lodging_detail,
+          notes: "Wait for client-rendered lodging detail cues without bypassing access controls.",
+          tests: "spec/fetch_util/browser_stabilization_spec.rb"
         }.freeze
 
         private

data/lib/fetch_util/browser/site_stabilization/travel.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      autoload :LodgingStabilization, "fetch_util/browser/site_stabilization/travel/lodging"
+
+      module TravelStabilization
+        include LodgingStabilization
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization.rb

@@ -3,104 +3,17 @@
 module FetchUtil
   class Browser
     module SiteStabilization
-      autoload :AzureDevopsPrProductStateScript,
-               "fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script"
-      autoload :AzureDevopsPrCommitDetailsScript,
-               "fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script"
-      autoload :AzureDevopsPrRequestHelpersScript,
-               "fetch_util/browser/site_stabilization/azure_devops_pr_request_helpers_script"
-      autoload :AzureDevopsPrRequestStateScript,
-               "fetch_util/browser/site_stabilization/azure_devops_pr_request_state_script"
-      autoload :AzureDevopsPrState, "fetch_util/browser/site_stabilization/azure_devops_pr_state"
-      autoload :AzureDevopsPrStabilization,
-               "fetch_util/browser/site_stabilization/azure_devops_pr_stabilization"
-      autoload :BitbucketCloudThreadState,
-               "fetch_util/browser/site_stabilization/bitbucket_cloud_thread_state"
-      autoload :BitbucketCloudThreads, "fetch_util/browser/site_stabilization/bitbucket_cloud_threads"
-      autoload :BitbucketCloudPullResourceModules,
-               "fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_modules"
-      autoload :CommunityAndMarketplace, "fetch_util/browser/site_stabilization/community_and_marketplace"
-      autoload :FacebookStabilization, "fetch_util/browser/site_stabilization/facebook_stabilization"
-      autoload :GerritChangeProductStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_change_product_state_script"
-      autoload :GerritChangeRequestStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_change_request_state_script"
-      autoload :GerritChangeState, "fetch_util/browser/site_stabilization/gerrit_change_state"
-      autoload :GerritChangeStabilization,
-               "fetch_util/browser/site_stabilization/gerrit_change_stabilization"
-      autoload :GerritFileResourceProductStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_product_state_script"
-      autoload :GerritFileResourceRouteStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_route_state_script"
-      autoload :GerritFileResourceFetchStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_fetch_state_script"
-      autoload :GerritFileResourceValidationStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script"
-      autoload :GerritFileResourceRequestStateScript,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_request_state_script"
-      autoload :GerritFileResourceState,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_state"
-      autoload :GerritFileResourceStabilization,
-               "fetch_util/browser/site_stabilization/gerrit_file_resource_stabilization"
-      autoload :GithubPullResources, "fetch_util/browser/site_stabilization/github_pull_resources"
-      autoload :GithubPullResourceStabilization,
-               "fetch_util/browser/site_stabilization/github_pull_resource_stabilization"
-      autoload :GithubThreads, "fetch_util/browser/site_stabilization/github_threads"
-      autoload :GiteaFamilyThreadProductState,
-               "fetch_util/browser/site_stabilization/gitea_family_thread_product_state"
-      autoload :GiteaFamilyThreadTimelineState,
-               "fetch_util/browser/site_stabilization/gitea_family_thread_timeline_state"
-      autoload :GiteaFamilyStabilization,
-               "fetch_util/browser/site_stabilization/gitea_family_stabilization"
-      autoload :GiteaFamilyVisibilityScript,
-               "fetch_util/browser/site_stabilization/gitea_family_visibility_script"
-      autoload :GiteaFamilyThreads, "fetch_util/browser/site_stabilization/gitea_family_threads"
-      autoload :GiteaFamilyPullResourceProductState,
-               "fetch_util/browser/site_stabilization/gitea_family_pull_resource_product_state"
-      autoload :GiteaFamilyPullResourceState,
-               "fetch_util/browser/site_stabilization/gitea_family_pull_resource_state"
-      autoload :GiteaFamilyPullResources, "fetch_util/browser/site_stabilization/gitea_family_pull_resources"
-      autoload :GitlabRepo, "fetch_util/browser/site_stabilization/gitlab_repo"
-      autoload :GitlabMergeRequestResourceVisibilityScript,
-               "fetch_util/browser/site_stabilization/gitlab_merge_request_resource_visibility_script"
-      autoload :GitlabMergeRequestResourceStateScript,
-               "fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state_script"
-      autoload :GitlabMergeRequestResourceState,
-               "fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state"
-      autoload :GitlabMergeRequestResources, "fetch_util/browser/site_stabilization/gitlab_merge_request_resources"
-      autoload :GitlabStabilization, "fetch_util/browser/site_stabilization/gitlab_stabilization"
-      autoload :GitlabThreads, "fetch_util/browser/site_stabilization/gitlab_threads"
-      autoload :SocialPlatforms, "fetch_util/browser/site_stabilization/social_platforms"
-      autoload :TravelAndLodging, "fetch_util/browser/site_stabilization/travel_and_lodging"
+      autoload :CommunityStabilization, "fetch_util/browser/site_stabilization/communities"
+      autoload :ForgeStabilization, "fetch_util/browser/site_stabilization/forges"
+      autoload :MarketplaceStabilization, "fetch_util/browser/site_stabilization/marketplaces"
+      autoload :SocialStabilization, "fetch_util/browser/site_stabilization/social"
+      autoload :TravelStabilization, "fetch_util/browser/site_stabilization/travel"
 
-      include AzureDevopsPrState
-      include AzureDevopsPrStabilization
-      include BitbucketCloudThreadState
-      include BitbucketCloudThreads
-      include BitbucketCloudPullResourceModules
-      include CommunityAndMarketplace
-      include FacebookStabilization
-      include GerritChangeState
-      include GerritChangeStabilization
-      include GerritFileResourceState
-      include GerritFileResourceStabilization
-      include GithubPullResources
-      include GithubPullResourceStabilization
-      include GithubThreads
-      include GiteaFamilyVisibilityScript
-      include GiteaFamilyThreadProductState
-      include GiteaFamilyThreadTimelineState
-      include GiteaFamilyStabilization
-      include GiteaFamilyThreads
-      include GiteaFamilyPullResourceProductState
-      include GiteaFamilyPullResourceState
-      include GiteaFamilyPullResources
-      include GitlabRepo
-      include GitlabStabilization
-      include GitlabMergeRequestResources
-      include GitlabThreads
-      include SocialPlatforms
-      include TravelAndLodging
+      include CommunityStabilization
+      include ForgeStabilization
+      include MarketplaceStabilization
+      include SocialStabilization
+      include TravelStabilization
     end
   end
 end

data/lib/fetch_util/browser/stabilization/page_flow.rb

@@ -6,10 +6,10 @@ module FetchUtil
     module Stabilization
       module PageFlow
         PAGE_FLOW_STABILIZATION_PROFILES = [
-          SiteStabilization::CommunityAndMarketplace::COMMUNITY_MARKETPLACE_STABILIZATION_PROFILES[:stabilize_reddit],
-          SiteStabilization::SocialPlatforms::SOCIAL_PLATFORM_STABILIZATION_PROFILES[:stabilize_instagram],
-          SiteStabilization::SocialPlatforms::SOCIAL_PLATFORM_STABILIZATION_PROFILES[:stabilize_facebook],
-          SiteStabilization::CommunityAndMarketplace::COMMUNITY_MARKETPLACE_STABILIZATION_PROFILES[:stabilize_ebay_search],
+          SiteStabilization::RedditStabilization::REDDIT_STABILIZATION_PROFILE,
+          SiteStabilization::InstagramStabilization::INSTAGRAM_STABILIZATION_PROFILE,
+          SiteStabilization::FacebookStabilization::FACEBOOK_STABILIZATION_PROFILE,
+          SiteStabilization::EbayStabilization::EBAY_STABILIZATION_PROFILE,
           SiteStabilization::GithubThreads::GITHUB_THREAD_STABILIZATION_PROFILE,
           SiteStabilization::GithubPullResources::GITHUB_PULL_RESOURCE_STABILIZATION_PROFILE,
           SiteStabilization::GitlabThreads::GITLAB_THREAD_STABILIZATION_PROFILE,
@@ -21,7 +21,7 @@ module FetchUtil
           SiteStabilization::AzureDevopsPrStabilization::AZURE_DEVOPS_PR_STABILIZATION_PROFILE,
           SiteStabilization::GerritFileResourceStabilization::GERRIT_FILE_RESOURCE_STABILIZATION_PROFILE,
           SiteStabilization::GerritChangeStabilization::GERRIT_CHANGE_STABILIZATION_PROFILE,
-          SiteStabilization::TravelAndLodging::TRAVEL_LODGING_STABILIZATION_PROFILES[:stabilize_lodging_detail],
+          SiteStabilization::LodgingStabilization::LODGING_STABILIZATION_PROFILE,
           { host: "t.me", path_query: ->(uri) { uri.path.match?(%r{\A/s/[^/]+/\d+/?\z}) },
             strategy: :wait_for_telegram_message, notes: "Wait for the requested public Telegram preview message.",
             tests: "spec/fetch_util/browser_stabilization_spec.rb" },
@@ -95,6 +95,8 @@ module FetchUtil
 
         def matching_stabilization_profile(url, profiles)
           uri = URI.parse(url)
+          return nil unless uri.is_a?(URI::HTTP) && uri.host
+
           host = FetchUtil.strip_www_host(url)
           profiles.find { |profile| profile_match?(profile, uri, host) }
         rescue URI::InvalidURIError

data/lib/fetch_util/browser.rb

@@ -2,6 +2,7 @@
 
 require "ferrum"
 require "json"
+require "securerandom"
 require "uri"
 
 module FetchUtil
@@ -31,6 +32,8 @@ module FetchUtil
     DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " \
                          "(KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
     DEFAULT_ACCEPT_LANGUAGE = "en-US,en;q=0.9"
+    SHADOW_ROOT_ACCESS_TOKEN = SecureRandom.hex(32).freeze
+    SHADOW_ROOT_READER_PROPERTY = "__fetchUtilClosedShadowRootReaderV1_#{SecureRandom.hex(16)}".freeze
     SOCIAL_LOGIN_PHASE_WAIT = 0.3
     POST_CONSENT_IDLE_TIMEOUT = 3.0
     CONTENT_READY_MIN_LENGTH = 200
@@ -230,6 +233,9 @@ module FetchUtil
           raise unless page_loaded_enough?(page)
         end
         stabilize_page(page, loaded_page_url(page, url))
+        if page.current_url == "about:blank" && url != "about:blank"
+          raise Ferrum::Error, "Request to #{url} finished on about:blank without a public page"
+        end
         page
       rescue Ferrum::PendingConnectionsError, Ferrum::TimeoutError, Ferrum::Error => e
         close_page(page)

data/lib/fetch_util/extractor.rb

@@ -5,6 +5,9 @@ require "json"
 module FetchUtil
   class Extractor
     INLINE_ASSET_PATHS = %w[vendor/readability.js vendor/turndown.js extract.js].freeze
+    VENDOR_ASSET_PATHS = INLINE_ASSET_PATHS.first(2).freeze
+    VENDOR_GLOBAL_NAMES = %w[Readability TurndownService].freeze
+    PRIVATE_DELIVERY_SENTINEL = '"fetch-util:standalone-api:v1"'
     @asset_cache = {}
     @cache_mutex = Mutex.new
 
@@ -22,6 +25,7 @@ module FetchUtil
       @reader_mode = reader_mode
       @asset_root = (asset_root || File.join(__dir__, "assets")).dup.freeze
       @extraction_call = nil
+      @private_asset_script = nil
     end
 
     def extract(page)
@@ -36,14 +40,26 @@ module FetchUtil
     private
 
     def inject_assets(page)
-      page.add_script_tag(path: asset_path("vendor/readability.js"))
-      page.add_script_tag(path: asset_path("vendor/turndown.js"))
-      page.add_script_tag(path: asset_path("extract.js"))
+      INLINE_ASSET_PATHS.each do |relative_path|
+        page.add_script_tag(path: asset_path(relative_path))
+      end
+    end
+
+    def inject_vendor_assets(page)
+      VENDOR_ASSET_PATHS.each do |relative_path|
+        page.add_script_tag(path: asset_path(relative_path))
+      end
     end
 
-    def inject_assets_inline(page)
-      inline_asset_scripts.each do |script|
-        page.evaluate("#{script}\ntrue")
+    def inject_vendor_assets_inline(page)
+      inline_asset_scripts.first(2).zip(VENDOR_GLOBAL_NAMES).each do |script, global_name|
+        page.evaluate(<<~JAVASCRIPT)
+          (() => {
+            #{script}
+            window.#{global_name} = #{global_name};
+            return true;
+          })()
+        JAVASCRIPT
       end
     end
 
@@ -52,14 +68,15 @@ module FetchUtil
       original_timeout = page.timeout if timeout_supported
       page.timeout = [original_timeout.to_f, 60].max if timeout_supported
 
-      inject_assets(page)
+      register_cdp_closed_shadow_roots(page)
+      inject_vendor_assets(page)
       page.evaluate(extraction_call)
     rescue Ferrum::TimeoutError
       begin
         page.evaluate("window.stop && window.stop()")
       rescue Ferrum::Error
       end
-      inject_assets_inline(page)
+      inject_vendor_assets_inline(page)
       page.evaluate(extraction_call)
     ensure
       restore_page_timeout(page, original_timeout) if timeout_supported
@@ -71,8 +88,97 @@ module FetchUtil
       nil
     end
 
+    def register_cdp_closed_shadow_roots(page)
+      return unless page.respond_to?(:command)
+
+      document = page.command("DOM.getDocument", depth: -1, pierce: true).fetch("root")
+      shadow_root_pairs(document).each do |host, root|
+        register_cdp_closed_shadow_root(page, host, root)
+      end
+    rescue Ferrum::Error
+      nil
+    end
+
+    def shadow_root_pairs(document)
+      pairs = []
+      pending = [document]
+      until pending.empty?
+        node = pending.pop
+        shadows = Array(node["shadowRoots"])
+        shadows.each { |root| pairs << [node, root] if root["shadowRootType"] == "closed" }
+        pending.concat(Array(node["children"]), shadows)
+      end
+      pairs
+    end
+
+    def register_cdp_closed_shadow_root(page, host, root)
+      object_ids = []
+      [host, root].each do |node|
+        object_ids << page.command(
+          "DOM.resolveNode",
+          backendNodeId: node.fetch("backendNodeId")
+        ).dig("object", "objectId")
+      end
+      return unless object_ids.all?
+
+      page.command(
+        "Runtime.callFunctionOn",
+        objectId: object_ids.first,
+        functionDeclaration: closed_shadow_registration_function,
+        arguments: [
+          { objectId: object_ids.last },
+          { value: Browser::SHADOW_ROOT_READER_PROPERTY },
+          { value: Browser::SHADOW_ROOT_ACCESS_TOKEN }
+        ],
+        returnByValue: true
+      )
+    rescue Ferrum::Error
+      nil
+    ensure
+      Array(object_ids).compact.each do |object_id|
+        page.command("Runtime.releaseObject", objectId: object_id)
+      rescue Ferrum::Error
+        nil
+      end
+    end
+
+    def closed_shadow_registration_function
+      <<~JAVASCRIPT
+        function(root, property, token) {
+          const reader = window[property];
+          return typeof reader === "function" && reader(token, "set", this, root) === true;
+        }
+      JAVASCRIPT
+    end
+
     def extraction_call
-      @extraction_call ||= "window.FetchUtilExtract.extract(#{JSON.generate(reader_mode: @reader_mode)})"
+      @extraction_call ||= begin
+        options = JSON.generate(
+          reader_mode: @reader_mode,
+          shadow_root_token: Browser::SHADOW_ROOT_ACCESS_TOKEN,
+          shadow_root_reader_property: Browser::SHADOW_ROOT_READER_PROPERTY
+        )
+        <<~JAVASCRIPT
+          (() => {
+            let __fetchUtilPrivateApi = null;
+            const __fetchUtilDeliverExtractApi = api => { __fetchUtilPrivateApi = api; };
+            #{private_asset_script}
+            if (!__fetchUtilPrivateApi || typeof __fetchUtilPrivateApi.extract !== "function") return null;
+            return __fetchUtilPrivateApi.extract(#{options});
+          })()
+        JAVASCRIPT
+      end
+    end
+
+    def private_asset_script
+      @private_asset_script ||= begin
+        source = inline_asset_scripts.last
+        marker_index = source.index(PRIVATE_DELIVERY_SENTINEL)
+        duplicate_index = source.index(PRIVATE_DELIVERY_SENTINEL, marker_index.to_i + PRIVATE_DELIVERY_SENTINEL.length)
+        raise ExtractionError, "Private extraction delivery sentinel is missing or duplicated" if marker_index.nil? || duplicate_index
+
+        source.sub(PRIVATE_DELIVERY_SENTINEL, "__fetchUtilDeliverExtractApi").freeze
+      end
     end
 
     def inline_asset_scripts

data/lib/fetch_util/raw_docs_fallback.rb

@@ -300,7 +300,7 @@ module FetchUtil
     def markdown_from_root(root, title)
       sections = []
       blocks = root.css(BLOCK_SELECTOR).to_a
-      blocks.unshift(root) if %w[ul ol].include?(root.name)
+      blocks.unshift(root) if BLOCK_ELEMENTS.include?(root.name)
       blocks.each do |node|
         next if node.ancestors.any? { |ancestor| %w[ul ol].include?(ancestor.name) }
 
@@ -320,6 +320,10 @@ module FetchUtil
           sections << [fence, code, fence].join("\n")
         when "ul", "ol"
           sections << markdown_list(node)
+        when "li"
+          content = node.dup
+          content.css("ul, ol").remove
+          sections << "- #{clean_text(content.text)}"
         when "tr"
           cells = node.css("th, td").map { |cell| clean_text(cell.text) }.reject(&:empty?)
           sections << "- #{cells.join(": ")}" unless cells.empty?

data/lib/fetch_util/regulatory/cache_store.rb

@@ -3,12 +3,14 @@
 module FetchUtil
   class Regulatory
     module CacheStore
+      include CacheValidation
+
       private
 
-      def cache_fetch(key)
+      def cache_fetch(key, shape: nil)
         path = cache_file_path(key)
         cached = read_cache(path)
-        return cached if cached
+        return cached if cached && (!shape || cache_payload_matches?(cached, shape))
 
         payload, cacheable = yield
         begin
@@ -20,7 +22,7 @@ module FetchUtil
       end
 
       def fetch_record(key, uri, fallback: nil, require_success: true)
-        cache_fetch(key) do
+        cache_fetch(key, shape: fallback) do
           response, cacheable = record_response(uri, require_success: require_success)
           payload = response ? yield(response.body, response) : fallback
           [payload, cacheable]

data/lib/fetch_util/regulatory/cache_validation.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module CacheValidation
+      private
+
+      def cache_payload_matches?(payload, shape)
+        case shape
+        when Hash
+          payload.is_a?(Hash) && shape.all? do |key, value_shape|
+            payload.key?(key) && cache_payload_matches?(payload[key], value_shape)
+          end
+        when Array
+          payload.is_a?(Array)
+        when nil
+          payload.nil? || payload.is_a?(String)
+        else
+          payload.is_a?(shape.class)
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/tdm_policy.rb

@@ -20,7 +20,7 @@ module FetchUtil
       end
 
       def tdm_policy_record(url, target_origin:)
-        cache_fetch("tdmpolicy:#{origin_key(target_origin)}:#{url}") do
+        cache_fetch("tdmpolicy:#{origin_key(target_origin)}:#{url}", shape: { "signals" => [] }) do
           response, cacheable = safe_get(url)
           signals = []
           if response&.status&.between?(200, 299) && json_like_response?(response.headers, response.body)

data/lib/fetch_util/regulatory.rb

@@ -43,6 +43,7 @@ module FetchUtil
     autoload :SourceSelection, "fetch_util/regulatory/source_selection"
     autoload :Signals, "fetch_util/regulatory/signals"
     autoload :FetchRecords, "fetch_util/regulatory/fetch_records"
+    autoload :CacheValidation, "fetch_util/regulatory/cache_validation"
     autoload :CacheStore, "fetch_util/regulatory/cache_store"
     autoload :Robots, "fetch_util/regulatory/robots"
     autoload :RobotGlobs, "fetch_util/regulatory/robot_globs"

data/lib/fetch_util/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FetchUtil
-  VERSION = "0.6.2"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fetch_util
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.8.0
 platform: ruby
 authors:
 - hmdne
@@ -100,54 +100,62 @@ files:
 - lib/fetch_util/browser/navigation.rb
 - lib/fetch_util/browser/navigation/headers_and_readiness.rb
 - lib/fetch_util/browser/navigation/navigator_patch.rb
+- lib/fetch_util/browser/navigation/shadow_root_patch.rb
 - lib/fetch_util/browser/site_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script.rb
-- lib/fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script.rb
-- lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_helpers_script.rb
-- lib/fetch_util/browser/site_stabilization/azure_devops_pr_request_state_script.rb
-- lib/fetch_util/browser/site_stabilization/azure_devops_pr_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/azure_devops_pr_state.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_product_state.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_request_state.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_diff_state.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_modules.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_state.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resources.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_thread_state.rb
-- lib/fetch_util/browser/site_stabilization/bitbucket_cloud_threads.rb
-- lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb
-- lib/fetch_util/browser/site_stabilization/facebook_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_change_product_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_change_request_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_change_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_change_state.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_fetch_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_product_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_request_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_route_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_state.rb
-- lib/fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_product_state.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_pull_resource_state.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_pull_resources.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_thread_product_state.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_thread_timeline_state.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_threads.rb
-- lib/fetch_util/browser/site_stabilization/gitea_family_visibility_script.rb
-- lib/fetch_util/browser/site_stabilization/github_pull_resource_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/github_pull_resources.rb
-- lib/fetch_util/browser/site_stabilization/github_threads.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state_script.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resource_visibility_script.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_merge_request_resources.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_repo.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_stabilization.rb
-- lib/fetch_util/browser/site_stabilization/gitlab_threads.rb
-- lib/fetch_util/browser/site_stabilization/social_platforms.rb
-- lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb
+- lib/fetch_util/browser/site_stabilization/communities.rb
+- lib/fetch_util/browser/site_stabilization/communities/reddit.rb
+- lib/fetch_util/browser/site_stabilization/forges.rb
+- lib/fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/commit_details_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/product_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/request_helpers_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/request_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/stabilization.rb
+- lib/fetch_util/browser/site_stabilization/forges/azure_devops/pull_requests/state.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/pulls/diff/product_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/pulls/diff/request_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/pulls/diff/state.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/pulls/resource_modules.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/pulls/resource_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/pulls/resources.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/threads/index.rb
+- lib/fetch_util/browser/site_stabilization/forges/bitbucket/threads/state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/changes/product_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/changes/request_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/changes/stabilization.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/changes/state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/fetch_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/product_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/request_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/route_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/stabilization.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gerrit/resources/validation_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/pulls/product_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/pulls/resources.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/pulls/state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/stabilization.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/threads/index.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/threads/product_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/threads/timeline_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitea/visibility_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/github/pull_resource_stabilization.rb
+- lib/fetch_util/browser/site_stabilization/forges/github/pull_resources.rb
+- lib/fetch_util/browser/site_stabilization/forges/github/threads.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/merge_requests/resource_state.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/merge_requests/resource_state_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/merge_requests/resource_visibility_script.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/merge_requests/resources.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/repository.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/stabilization.rb
+- lib/fetch_util/browser/site_stabilization/forges/gitlab/threads.rb
+- lib/fetch_util/browser/site_stabilization/forges/observation_state.rb
+- lib/fetch_util/browser/site_stabilization/marketplaces.rb
+- lib/fetch_util/browser/site_stabilization/marketplaces/ebay.rb
+- lib/fetch_util/browser/site_stabilization/social.rb
+- lib/fetch_util/browser/site_stabilization/social/facebook.rb
+- lib/fetch_util/browser/site_stabilization/social/instagram.rb
+- lib/fetch_util/browser/site_stabilization/travel.rb
+- lib/fetch_util/browser/site_stabilization/travel/lodging.rb
 - lib/fetch_util/browser/stabilization.rb
 - lib/fetch_util/browser/stabilization/anubis.rb
 - lib/fetch_util/browser/stabilization/page_flow.rb
@@ -160,6 +168,7 @@ files:
 - lib/fetch_util/raw_docs_fallback.rb
 - lib/fetch_util/regulatory.rb
 - lib/fetch_util/regulatory/cache_store.rb
+- lib/fetch_util/regulatory/cache_validation.rb
 - lib/fetch_util/regulatory/directives.rb
 - lib/fetch_util/regulatory/fetch_records.rb
 - lib/fetch_util/regulatory/headers.rb