fetch_util 0.3.2 → 0.4.0

data/lib/fetch_util/result.rb

@@ -3,10 +3,13 @@
 module FetchUtil
   class Result
     attr_reader :url, :final_url, :title, :byline, :excerpt, :site_name,
-                :published_time, :canonical_url, :language, :html, :markdown,
+                :published_time, :canonical_url, :language, :name, :company, :location,
+                :description, :ingredients, :instructions, :bedrooms, :bathrooms,
+                :area_sqft, :html, :markdown,
                 :metadata, :reader_mode, :content_type, :suspect, :warnings,
                 :content_completeness_ratio, :content_format, :paywall_state,
-                :error_message
+                :price, :rating, :address, :social_kind, :platform, :handle,
+                :reply_count, :community, :score, :error_message
 
     class << self
       def from_payload(url:, final_url:, payload:, canonical_url:, content_type:, warnings:, suspect:)
@@ -22,7 +25,25 @@ module FetchUtil
           warnings: warnings,
           content_completeness_ratio: content_completeness_ratio,
           content_format: content_format,
-          paywall_state: paywall_state
+          paywall_state: paywall_state,
+          name: payload["name"],
+          company: payload["company"],
+          location: payload["location"],
+          description: payload["description"],
+          ingredients: payload["ingredients"],
+          instructions: payload["instructions"],
+          bedrooms: payload["bedrooms"],
+          bathrooms: payload["bathrooms"],
+          area_sqft: payload["areaSqft"],
+          price: payload["price"],
+          rating: payload["rating"],
+          address: payload["address"],
+          social_kind: payload["socialKind"],
+          platform: payload["platform"],
+          handle: payload["handle"],
+          reply_count: payload["replyCount"],
+          community: payload["community"],
+          score: payload["score"]
         )
 
         new(
@@ -35,6 +56,15 @@ module FetchUtil
           published_time: payload["publishedTime"],
           canonical_url: canonical_url,
           language: payload["language"],
+          name: payload["name"],
+          company: payload["company"],
+          location: payload["location"],
+          description: payload["description"],
+          ingredients: payload["ingredients"],
+          instructions: payload["instructions"],
+          bedrooms: payload["bedrooms"],
+          bathrooms: payload["bathrooms"],
+          area_sqft: payload["areaSqft"],
           html: payload["html"],
           markdown: payload["markdown"],
           metadata: metadata,
@@ -44,7 +74,16 @@ module FetchUtil
           warnings: warnings,
           content_completeness_ratio: content_completeness_ratio,
           content_format: content_format,
-          paywall_state: paywall_state
+          paywall_state: paywall_state,
+          price: payload["price"],
+          rating: payload["rating"],
+          address: payload["address"],
+          social_kind: payload["socialKind"],
+          platform: payload["platform"],
+          handle: payload["handle"],
+          reply_count: payload["replyCount"],
+          community: payload["community"],
+          score: payload["score"]
         )
       end
 
@@ -54,7 +93,13 @@ module FetchUtil
           content_type: "error",
           suspect: true,
           warnings: [warning],
-          error_message: message
+          error_message: message,
+          social_kind: nil,
+          platform: nil,
+          handle: nil,
+          reply_count: nil,
+          community: nil,
+          score: nil
         }.freeze
 
         new(
@@ -67,6 +112,15 @@ module FetchUtil
           published_time: nil,
           canonical_url: nil,
           language: nil,
+          name: nil,
+          company: nil,
+          location: nil,
+          description: nil,
+          ingredients: nil,
+          instructions: nil,
+          bedrooms: nil,
+          bathrooms: nil,
+          area_sqft: nil,
           html: nil,
           markdown: "",
           metadata: metadata,
@@ -81,7 +135,10 @@ module FetchUtil
       private
 
       def payload_metadata(payload, canonical_url:, final_url:, content_type:, suspect:, warnings:,
-                           content_completeness_ratio:, content_format:, paywall_state:)
+                           content_completeness_ratio:, content_format:, paywall_state:,
+                           name:, company:, location:, description:, ingredients:, instructions:,
+                           bedrooms:, bathrooms:, area_sqft:, price:, rating:, address:,
+                           social_kind:, platform:, handle:, reply_count:, community:, score:)
         {
           title: payload["title"],
           byline: payload["byline"],
@@ -90,6 +147,15 @@ module FetchUtil
           published_time: payload["publishedTime"],
           canonical_url: canonical_url,
           language: payload["language"],
+          name: name,
+          company: company,
+          location: location,
+          description: description,
+          ingredients: ingredients,
+          instructions: instructions,
+          bedrooms: bedrooms,
+          bathrooms: bathrooms,
+          area_sqft: area_sqft,
           content_url: final_url,
           reader_mode: payload["readerMode"],
           content_type: content_type,
@@ -97,14 +163,27 @@ module FetchUtil
           warnings: warnings,
           content_completeness_ratio: content_completeness_ratio,
           content_format: content_format,
-          paywall_state: paywall_state
+          paywall_state: paywall_state,
+          price: price,
+          rating: rating,
+          address: address,
+          social_kind: social_kind,
+          platform: platform,
+          handle: handle,
+          reply_count: reply_count,
+          community: community,
+          score: score
         }.freeze
       end
     end
 
     def initialize(url:, final_url:, title:, byline:, excerpt:, site_name:, published_time:,
-                   canonical_url:, language:, html:, markdown:, metadata:, reader_mode:, content_type:, suspect:, warnings:,
-                   content_completeness_ratio: 1.0, content_format: nil, paywall_state: nil, error_message: nil)
+                   canonical_url:, language:, name:, company:, location:, description:, ingredients:,
+                   instructions:, bedrooms:, bathrooms:, area_sqft:, html:, markdown:, metadata:,
+                   reader_mode:, content_type:, suspect:, warnings:,
+                   content_completeness_ratio: 1.0, content_format: nil, paywall_state: nil,
+                   price: nil, rating: nil, address: nil, social_kind: nil, platform: nil,
+                   handle: nil, reply_count: nil, community: nil, score: nil, error_message: nil)
       @url = url
       @final_url = final_url
       @title = title
@@ -114,6 +193,15 @@ module FetchUtil
       @published_time = published_time
       @canonical_url = canonical_url
       @language = language
+      @name = name
+      @company = company
+      @location = location
+      @description = description
+      @ingredients = ingredients
+      @instructions = instructions
+      @bedrooms = bedrooms
+      @bathrooms = bathrooms
+      @area_sqft = area_sqft
       @html = html
       @markdown = markdown
       @metadata = metadata.freeze
@@ -124,6 +212,15 @@ module FetchUtil
       @content_completeness_ratio = content_completeness_ratio
       @content_format = content_format&.freeze
       @paywall_state = paywall_state&.freeze
+      @price = price
+      @rating = rating
+      @address = address
+      @social_kind = social_kind
+      @platform = platform
+      @handle = handle
+      @reply_count = reply_count
+      @community = community
+      @score = score
       @error_message = error_message
     end
 
@@ -138,6 +235,15 @@ module FetchUtil
         published_time: published_time,
         canonical_url: canonical_url,
         language: language,
+        name: name,
+        company: company,
+        location: location,
+        description: description,
+        ingredients: ingredients,
+        instructions: instructions,
+        bedrooms: bedrooms,
+        bathrooms: bathrooms,
+        area_sqft: area_sqft,
         html: html,
         markdown: markdown,
         metadata: metadata,
@@ -148,6 +254,15 @@ module FetchUtil
         content_completeness_ratio: content_completeness_ratio,
         content_format: content_format,
         paywall_state: paywall_state,
+        price: price,
+        rating: rating,
+        address: address,
+        social_kind: social_kind,
+        platform: platform,
+        handle: handle,
+        reply_count: reply_count,
+        community: community,
+        score: score,
         error_message: error_message
       }
     end

data/lib/fetch_util/searcher.rb

@@ -5,8 +5,6 @@ require "uri"
 
 module FetchUtil
   class Searcher
-    MAX_SNIPPET_LENGTH = 180
-
     SOURCES = {
       "duckduckgo" => "https://duckduckgo.com/?q=%<query>s&ia=web&kl=us-en",
       "google" => "https://www.google.com/search?hl=en&q=%<query>s",
@@ -21,10 +19,10 @@ module FetchUtil
     include ResultFiltering
     private_constant :ResultFiltering
 
-    def initialize(fetcher: nil, request_log: RequestLog.new, sources: nil, limit: 10, concurrency: 2, verbose: false, **fetch_options)
+    def initialize(fetcher: nil, request_log: RequestLog.new, sources: nil, limit: nil, concurrency: 2, verbose: false, **fetch_options)
       @request_log = request_log
       @sources = Array(sources || DEFAULT_SOURCES).map(&:to_s)
-      @limit = limit.to_i
+      @limit = limit
       @verbose = verbose
       @fetcher = fetcher || ParallelFetcher.new(concurrency: concurrency, request_log: request_log, **fetch_options)
     end
@@ -45,7 +43,7 @@ module FetchUtil
 
       {
         query: encoded_query,
-        results: formatted_results(aggregate(urls.keys, fetched).first(limit))
+        results: formatted_results(apply_limit(aggregate(urls.keys, fetched)))
       }
     end
 
@@ -53,6 +51,10 @@ module FetchUtil
 
     attr_reader :limit
 
+    def apply_limit(results)
+      limit.nil? ? results : results.first(limit)
+    end
+
     def search_urls(query)
       urls = {}
 
@@ -131,7 +133,7 @@ module FetchUtil
           url: result[:url]
         }
         item[:snippet] = result[:snippet] if result[:snippet]
-        if verbose?
+        if @verbose
           item[:sources] = result[:sources]
           item[:ranks] = result[:ranks]
         end
@@ -204,10 +206,6 @@ module FetchUtil
       item
     end
 
-    def verbose?
-      @verbose
-    end
-
     def compact_text(value)
       FetchUtil.normalize_whitespace(value)
     end
@@ -296,13 +294,7 @@ module FetchUtil
       return nil if metadata_only_snippet?(text)
       return nil if jammed_navigation_text?(text)
 
-      truncate(text, MAX_SNIPPET_LENGTH)
-    end
-
-    def truncate(text, max_length)
-      return text if text.length <= max_length
-
-      "#{text[0, max_length - 3].rstrip}..."
+      text
     end
 
     def domain_only?(text, host)

data/lib/fetch_util/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FetchUtil
-  VERSION = "0.3.2"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fetch_util
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - hmdne
@@ -84,6 +84,7 @@ files:
 - lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb
 - lib/fetch_util/browser/site_stabilization/gitlab_repo.rb
 - lib/fetch_util/browser/site_stabilization/social_platforms.rb
+- lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb
 - lib/fetch_util/browser/stabilization.rb
 - lib/fetch_util/browser/stabilization/page_flow.rb
 - lib/fetch_util/browser/stabilization/spa_hydration.rb