fetch_util 0.3.0 → 0.3.1

data/lib/fetch_util/raw_docs_fallback.rb

@@ -20,6 +20,7 @@ module FetchUtil
       "nav",
       "aside",
       "footer",
+      ".discord",
       ".toc",
       ".sidebar",
       ".breadcrumbs",
@@ -29,10 +30,18 @@ module FetchUtil
       "button"
     ].freeze
     DOCS_ROOT_SELECTORS = [
+      "main .sl-markdown-content",
+      "main [data-pagefind-body]",
+      ".sl-markdown-content",
+      "[data-pagefind-body]",
       "main article",
       "main",
       "article",
       "[role='main']",
+      "[class*='content']",
+      "[id*='content']",
+      "[class*='article']",
+      "[id*='article']",
       ".content",
       ".resource-container"
     ].freeze
@@ -58,7 +67,9 @@ module FetchUtil
       return nil unless root
 
       prune!(root)
-      title = clean_text(fragment_title(document, final_url) || first_heading(root) || meta_title(document) || document.title)
+      title = [fragment_title(document, final_url), first_heading(root), meta_title(document), document.title]
+              .map { |candidate| clean_text(candidate) }
+              .find { |candidate| !candidate.empty? }
       markdown = markdown_from_root(root, title)
       return nil if clean_text(markdown).length < 40
 
@@ -149,23 +160,49 @@ module FetchUtil
     end
 
     def docs_root(document)
+      candidates = []
       DOCS_ROOT_SELECTORS.each do |selector|
-        node = document.at_css(selector)
-        return node.dup if node && clean_text(node.text).length >= 120
+        document.css(selector).each do |node|
+          text = clean_text(node.text)
+          next if text.length < 120
+
+          candidates << [node, text.length + (node.css("p").length * 200)]
+        end
       end
 
+      best = candidates.max_by { |_node, score| score }
+      return best.first.dup if best
+
       body = document.at_css("body")
       body&.dup
     end
 
     def prune!(root)
       DROP_SELECTORS.each { |selector| root.css(selector).remove }
+      prune_leading_promo_cards!(root)
       root.css("*").each do |node|
         text = clean_text(node.text)
         node.remove if text.match?(PRUNED_TEXT_PATTERN)
       end
     end
 
+    def prune_leading_promo_cards!(root)
+      return unless root.at_css("h1, h2")
+
+      root.css("article.card, .card").each do |node|
+        text = clean_text(node.text)
+        next if node.ancestors.any? { |ancestor| class_list(ancestor).any? { |klass| %w[landing-card card--fullwidth].include?(klass) } }
+        next if node.at_css(".title, .body")
+        next if text.empty? || text.length > 600 || node.at_css("h1, h2, h3, h4, h5, h6")
+
+        node.remove
+      end
+    end
+
+    def class_list(node)
+      node["class"].to_s.split
+    end
+
     def meta_title(document)
       meta_value(document, "og:title", attr: "property") || document.title
     end

data/lib/fetch_util/result.rb

@@ -5,11 +5,106 @@ module FetchUtil
     attr_reader :url, :final_url, :title, :byline, :excerpt, :site_name,
                 :published_time, :canonical_url, :language, :html, :markdown,
                 :metadata, :reader_mode, :content_type, :suspect, :warnings,
-                :content_completeness_ratio, :content_format, :paywall_state
+                :content_completeness_ratio, :content_format, :paywall_state,
+                :error_message
+
+    class << self
+      def from_payload(url:, final_url:, payload:, canonical_url:, content_type:, warnings:, suspect:)
+        content_completeness_ratio = payload["contentCompletenessRatio"]&.to_f || 1.0
+        content_format = payload["contentFormat"]
+        paywall_state = payload["paywallState"]
+        metadata = payload_metadata(
+          payload,
+          canonical_url: canonical_url,
+          final_url: final_url,
+          content_type: content_type,
+          suspect: suspect,
+          warnings: warnings,
+          content_completeness_ratio: content_completeness_ratio,
+          content_format: content_format,
+          paywall_state: paywall_state
+        )
+
+        new(
+          url: url,
+          final_url: final_url,
+          title: payload["title"],
+          byline: payload["byline"],
+          excerpt: payload["excerpt"],
+          site_name: payload["siteName"],
+          published_time: payload["publishedTime"],
+          canonical_url: canonical_url,
+          language: payload["language"],
+          html: payload["html"],
+          markdown: payload["markdown"],
+          metadata: metadata,
+          reader_mode: payload["readerMode"],
+          content_type: content_type,
+          suspect: suspect,
+          warnings: warnings,
+          content_completeness_ratio: content_completeness_ratio,
+          content_format: content_format,
+          paywall_state: paywall_state
+        )
+      end
+
+      def error(url:, warning:, message:)
+        metadata = {
+          content_url: url,
+          content_type: "error",
+          suspect: true,
+          warnings: [warning],
+          error_message: message
+        }.freeze
+
+        new(
+          url: url,
+          final_url: url,
+          title: nil,
+          byline: nil,
+          excerpt: nil,
+          site_name: nil,
+          published_time: nil,
+          canonical_url: nil,
+          language: nil,
+          html: nil,
+          markdown: "",
+          metadata: metadata,
+          reader_mode: nil,
+          content_type: "error",
+          suspect: true,
+          warnings: [warning],
+          error_message: message
+        )
+      end
+
+      private
+
+      def payload_metadata(payload, canonical_url:, final_url:, content_type:, suspect:, warnings:,
+                           content_completeness_ratio:, content_format:, paywall_state:)
+        {
+          title: payload["title"],
+          byline: payload["byline"],
+          excerpt: payload["excerpt"],
+          site_name: payload["siteName"],
+          published_time: payload["publishedTime"],
+          canonical_url: canonical_url,
+          language: payload["language"],
+          content_url: final_url,
+          reader_mode: payload["readerMode"],
+          content_type: content_type,
+          suspect: suspect,
+          warnings: warnings,
+          content_completeness_ratio: content_completeness_ratio,
+          content_format: content_format,
+          paywall_state: paywall_state
+        }.freeze
+      end
+    end
 
     def initialize(url:, final_url:, title:, byline:, excerpt:, site_name:, published_time:,
                    canonical_url:, language:, html:, markdown:, metadata:, reader_mode:, content_type:, suspect:, warnings:,
-                   content_completeness_ratio: 1.0, content_format: nil, paywall_state: nil)
+                   content_completeness_ratio: 1.0, content_format: nil, paywall_state: nil, error_message: nil)
       @url = url
       @final_url = final_url
       @title = title
@@ -29,6 +124,7 @@ module FetchUtil
       @content_completeness_ratio = content_completeness_ratio
       @content_format = content_format&.freeze
       @paywall_state = paywall_state&.freeze
+      @error_message = error_message
     end
 
     def to_h
@@ -51,7 +147,8 @@ module FetchUtil
         warnings: warnings,
         content_completeness_ratio: content_completeness_ratio,
         content_format: content_format,
-        paywall_state: paywall_state
+        paywall_state: paywall_state,
+        error_message: error_message
       }
     end
   end

data/lib/fetch_util/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FetchUtil
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fetch_util
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - hmdne
@@ -82,6 +82,7 @@ files:
 - lib/fetch_util/browser/navigation/navigator_patch.rb
 - lib/fetch_util/browser/site_stabilization.rb
 - lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb
+- lib/fetch_util/browser/site_stabilization/gitlab_repo.rb
 - lib/fetch_util/browser/site_stabilization/social_platforms.rb
 - lib/fetch_util/browser/stabilization.rb
 - lib/fetch_util/browser/stabilization/page_flow.rb