scrapedo 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c9655b79b553adb3cff6ef04a7fa82755080b89c33253c41fa0d0e0b404a602
-  data.tar.gz: d273348cb935ad6b6fa5848f02673614dde8eb0518f423919267ffe6fa313bf9
+  metadata.gz: 668a016977a4780253fc501ca7a95e080fbb4d8d5e19d6ba57feb8b427dbec18
+  data.tar.gz: 215c25a17b97fdc39bb2532cde698d885619d9b63b83adb0de124d9b9600dcd3
 SHA512:
-  metadata.gz: 38b9c7a09663c093dd4db35b16e807855d3a49043854237fd79db06cde655fb87a66129d85af8c300c5a73c49bcdf6f90c05f89b6247ba03233ba423aee81fbc
-  data.tar.gz: 1eacbae9f1c0c94601f534273a8c52d00c1eff683f31869ea7754d347e42f7581eb36921530ef576c4857548afe895ffc2486687e030a1c31a9efe27bac0e6cd
+  metadata.gz: 4ae653db01a38e250df1d5b4a5fd54aec1e14e8e0f57dd274cd632b4d71b9ee747598f2054e3431ba28a40b931d271b4ec7250ab5f62826d876960ca4ce80eaa
+  data.tar.gz: a326ae65643a30751d3423069a1f1ad5fa62cc8191717fe6da7e4d59b47270cf66b1f5287c58678d1a005459a022509f1c2076e3171efc1d82d6bfb6b2d98163

data/.rubocop.yml

@@ -1,5 +1,6 @@
 AllCops:
   TargetRubyVersion: 3.0
+  SuggestExtensions: false
 
 Style/StringLiterals:
   Enabled: true
@@ -11,3 +12,18 @@ Style/StringLiteralsInInterpolation:
 
 Layout/LineLength:
   Max: 120
+  AllowedPatterns:
+    - !ruby/regexp /\A\s*#/
+  Exclude:
+    - '*.gemspec'
+
+Lint/AmbiguousBlockAssociation:
+  Description: This cop checks for ambiguous block association with method when param passed without parentheses.
+  Enabled: false
+
+Style/Documentation:
+  Description: This cop checks for missing top-level documentation of classes and modules.
+  Enabled: false
+
+Metrics/ClassLength:
+  Max: 150

data/README.md

@@ -5,6 +5,8 @@
 
 An unofficial Ruby gem designed to provide a convenient interface for Ruby developers. It eliminates the need to handle complex HTTP requests and parsing logic, allowing you to easily access data from the Scrapedo API (currently supports only Google Search API) within your applications.
 
+It supports **ALL** parameters of the [Google Search API](https://scrape.do/documentation/google-search-api/search/).
+
 ## Installation
 
 Install the gem and add to the application's Gemfile by executing:

data/lib/scrapedo/google_builder.rb

@@ -5,6 +5,12 @@ require "json"
 require "yaml"
 
 class GoogleBuilder
+  # @!method desktop
+  #   Desktop SERP layout. See {General Parameters}[https://scrape.do/documentation/google-search-api/search/#general].
+  #
+  # @!method mobile
+  #   Mobile SERP layout. See {General Parameters}[https://scrape.do/documentation/google-search-api/search/#general].
+
   CONFIG_PATH = Pathname.new(__dir__ || ".").join("config/google_builder").expand_path
   FILE_MAP = {
     time_period: "time_period.yml",
@@ -15,6 +21,10 @@ class GoogleBuilder
     google_domain: "domain.yml"
   }.freeze
 
+  # Initializes a new instance of GoogleBuilder
+  #
+  # @param token [String]
+  # @param all_params [Boolean] If true, all available parameters will be included, otherwise only common parameters will be included.
   def initialize(token, all_params: false)
     @has_next = false
     @data = { token: token }
@@ -22,10 +32,16 @@ class GoogleBuilder
     @all_params = all_params
   end
 
+  # Returns +true+ if result have more pages; otherwise +false+.
   def next?
     @has_next
   end
 
+  # Search query. See {Required Parameters}[https://scrape.do/documentation/google-search-api/search/#required].
+  #
+  # @note This is required!
+  # @param query [String]
+  # @raise ArgumentError if query is nil or empty.
   def search(query)
     raise ArgumentError, "Query is required" if blank? query
 
@@ -35,71 +51,154 @@ class GoogleBuilder
     self
   end
 
+  # Device type for SERP layout. See {General Parameters}[https://scrape.do/documentation/google-search-api/search/#general].
+  # Accepted values: desktop, mobile, default is desktop.
+  #
+  # This can also use GoogleBuilder#{device}. For example:
+  # - GoogleBuilder#device("desktop") is same as GoogleBuilder#desktop.
+  # - GoogleBuilder#device("mobile") is same as GoogleBuilder#mobile.
+  #
+  # @param device [String]
   def device(device)
     @data[:device] = device if %w[desktop mobile].include?(device)
     self
   end
 
+  # Result includes the raw Google HTML. Useful for debugging and custom parsing.
+  # See {General Parameters}[https://scrape.do/documentation/google-search-api/search/#general].
   def include_html
     @data[:include_html] = true
     self
   end
 
+  # Host language of the Google UI. Default is en.
+  # Supports 150+ languages. ISO 639-1 codes. See {Language Parameter}[https://scrape.do/documentation/google-search-api/localization/#language-parameter-hl].
+  #
+  # This can also use GoogleBuilder#hl_{language}. For example:
+  # - GoogleBuilder#hl("en") is same as GoogleBuilder#hl_en and GoogleBuilder#hl_english.
+  # - GoogleBuilder#hl("ach") is same as GoogleBuilder#hl_ach and GoogleBuilder#hl_luo. (For this one, all_params must be true)
+  #
+  # GoogleBuilder#host_language is an alias for GoogleBuilder#hl.
+  #
+  # @param language [String]
   def hl(language)
     @data[:hl] = language
     self
   end
   alias host_language hl
 
+  # Geo Location (datacenter country). Determines from which country's perspective results are ranked and returned. Default is us.
+  # Supports 240+ countries. ISO 3166-1 alpha-2 codes. See {Country Parameter}[https://scrape.do/documentation/google-search-api/localization/#country-parameter-gl].
+  #
+  # This can also use GoogleBuilder#gl_{location}. For example:
+  # - GoogleBuilder#gl("us") is same as GoogleBuilder#gl_us and GoogleBuilder#gl_united_states.
+  # - GoogleBuilder#gl("cl") is same as GoogleBuilder#gl_cl and GoogleBuilder#gl_chile. (For this one, all_params must be true)
+  #
+  # GoogleBuilder#geo_location is an alias for GoogleBuilder#gl.
+  #
+  # @param location [String]
   def gl(location)
     @data[:gl] = location
     self
   end
   alias geo_location gl
 
-  def google_domain(domain)
+  # Google domain to query. Prefixes [https://], [http://], and [www.] are automatically stripped. Default is google.com.
+  # Supports 84 regional domains. See {Supported Google Domains}[https://scrape.do/documentation/google-search-api/localization/#supported-google-domains].
+  #
+  # This can also use GoogleBuilder#domain_{location}. For example:
+  # - GoogleBuilder#domain("google.com") is same as GoogleBuilder#domain_united_states.
+  #
+  # GoogleBuilder#google_domain is an alias for GoogleBuilder#domain.
+  #
+  # @param domain [String]
+  def domain(domain)
     @data[:google_domain] = domain
     self
   end
-  alias domain google_domain
+  alias google_domain domain
 
+  # Location name in Google's canonical format. Automatically encoded to UULE internally.
+  # See {Location}[https://scrape.do/documentation/google-search-api/localization/#location].
+  #
+  # Format: City,State/Region,Country
+  #
+  # Examples:
+  #   [Istanbul,Istanbul,Turkey]
+  #   [New York,New York,United States]
+  #
+  # @param location [String]
   def location(location)
     @data[:location] = location
     self
   end
 
+  # Google UULE-encoded location string.
+  # Auto-generated from location when not provided.
+  # If both location and uule are sent, uule takes priority. location is sufficient for most use cases.
+  # See {uule}[https://scrape.do/documentation/google-search-api/localization/#uule].
+  #
+  # @param uule [String]
   def uule(uule)
     @data[:uule] = uule
     self
   end
 
+  # Language Restrict applies strict language filtering. Only results written in the specified language are returned.
+  # Supports 35 languages. See {Language Restrict Parameter}[https://scrape.do/documentation/google-search-api/localization/#language-restrict-parameter-lr].
+  #
+  # This can also use GoogleBuilder#lr_{language}. For example:
+  # - GoogleBuilder#lr("en") is same as GoogleBuilder#lr_en and GoogleBuilder#lr_english.
+  #
+  # GoogleBuilder#language_restrict is an alias for GoogleBuilder#lr.
+  #
+  # @param language [String]
   def lr(language)
     @data[:lr] = language
     self
   end
   alias language_restrict lr
 
+  # Country Restrict applies strict country filtering. Only results originating from the specified country are returned.
+  # Supports 240+ countries. See {Country Restrict Parameter}[https://scrape.do/documentation/google-search-api/localization/#country-restrict-parameter-cr].
+  #
+  # This can also use GoogleBuilder#cr_{country}. For example:
+  # - GoogleBuilder#cr("countryUS") is same as GoogleBuilder#cr_us and GoogleBuilder#cr_united_states.
+  # - GoogleBuilder#cr("countryAL") is same as GoogleBuilder#cr_al and GoogleBuilder#cr_albania. (For this one, all_params must be true)
+  #
+  # GoogleBuilder#country_restrict is an alias for GoogleBuilder#cr.
+  #
+  # @param country [String]
   def cr(country)
     @data[:cr] = country
     self
   end
   alias country_restrict cr
 
+  # Send active to filter adult content from results.
   def safe_search
     @data[:safe] = "active"
     self
   end
 
+  # Enables Google's automatic spelling correction.
   def enable_nfpr
     @data[:nfpr] = true
     self
   end
 
+  # Disables "Similar Results" and "Omitted Results" filters.
   def disable_filter
     @data[:filter] = false
     self
   end
 
+  # Start scraping from a specific offset.
+  # See {General Parameters}[https://scrape.do/documentation/google-search-api/search/#general].
+  #
+  # @param start [Integer] The offset to start from. Default is 0.
+  # @raise ArgumentError If the search query is not set.
+  # @return [JSON] The search results starting from the specified offset.
   def start(start = 0)
     raise ArgumentError, "Query is required" if blank? @data[:q]
 
@@ -110,6 +209,9 @@ class GoogleBuilder
     result
   end
 
+  # Gets next page if result has next page.
+  #
+  # @return [JSON] The next page of results.
   def next
     start(@data[:start] + 10) if next?
   end
@@ -121,6 +223,9 @@ class GoogleBuilder
     end
   end
 
+  # Set any parameter from {Google Search API}[https://scrape.do/documentation/google-search-api/search/].
+  #
+  # @param params [Hash] A hash of parameters to set.
   def params(*params)
     params.first.each do |key, value|
       key_sym = key.to_sym
@@ -129,6 +234,7 @@ class GoogleBuilder
     self
   end
 
+  # Returns the URL to be used for the request.
   def scrapedo_url
     url = URI("https://api.scrape.do/plugin/google/search")
     url.query = URI.encode_www_form(@data)

data/lib/scrapedo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Scrapedo
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/scrapedo.rb

@@ -4,9 +4,16 @@ require_relative "scrapedo/version"
 require_relative "scrapedo/google_builder"
 
 module Scrapedo
-  def self.google(token)
+  # Generates Google Search Endpoint. See {Scrapedo API}[https://scrape.do/documentation/google-search-api/search/]
+  # @param token [String]
+  # @param all_params [Boolean] If true, all available parameters will be included, otherwise only common parameters will be included.
+  # @raise ArgumentError if token is nil or empty.
+  # @example
+  #   Scrapedo.google("your_token")
+  #   Scrapedo.google("your_token", true)
+  def self.google(token, all_params: false)
     raise ArgumentError, "Token is required" if token.nil? || token.empty?
 
-    GoogleBuilder.new(token)
+    GoogleBuilder.new(token, all_params: all_params)
   end
 end

data/scrapedo.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.email = ["boss.yuan@outlook.com"]
 
   spec.summary = "Ruby Gem For Scrape.do"
-  spec.description = "An unofficial Ruby gem designed to provide a convenient interface for Ruby developers. It eliminates the need to handle complex HTTP requests and parsing logic, allowing you to easily access data from the Scrapedo API (currently supports only Google Search API) within your applications."
+  spec.description = "An unofficial Ruby gem designed to provide a convenient interface for Ruby developers. It eliminates the need to handle complex HTTP requests and parsing logic, allowing you to easily access data from the Scrapedo API (currently supports only Google Search API) within your applications. It supports all parameters of the [Google Search API](https://scrape.do/documentation/google-search-api/search/)."
   spec.homepage = "https://github.com/hypersport/scrapedo"
   spec.license = "MIT"
   spec.required_ruby_version = ">= 3.0.0"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrapedo
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - hypersport
@@ -13,7 +13,8 @@ dependencies: []
 description: An unofficial Ruby gem designed to provide a convenient interface for
   Ruby developers. It eliminates the need to handle complex HTTP requests and parsing
   logic, allowing you to easily access data from the Scrapedo API (currently supports
-  only Google Search API) within your applications.
+  only Google Search API) within your applications. It supports all parameters of
+  the [Google Search API](https://scrape.do/documentation/google-search-api/search/).
 email:
 - boss.yuan@outlook.com
 executables: []
@@ -36,7 +37,6 @@ files:
 - lib/scrapedo/google_builder.rb
 - lib/scrapedo/version.rb
 - scrapedo.gemspec
-- sig/scrapedo.rbs
 homepage: https://github.com/hypersport/scrapedo
 licenses:
 - MIT

data/sig/scrapedo.rbs

@@ -1,66 +0,0 @@
-module Scrapedo
-  VERSION: String
-  def self.google: -> GoogleBuilder
-end
-
-class GoogleBuilder
-  CONFIG_PATH: Pathname
-  FILE_MAP: Hash[Symbol, String]
-
-  @has_next: bool
-  @data: Hash[Object, Object]
-  @params_cache: Hash[Object, Object]
-  @all_params: bool
-
-  def initialize: (String) -> void
-
-  def next?: () -> bool
-
-  def search: (String) -> GoogleBuilder
-
-  def device: -> GoogleBuilder
-
-  def include_html: -> GoogleBuilder
-
-  def hl: -> GoogleBuilder
-  alias host_language hl
-
-  def gl: -> GoogleBuilder
-  alias geo_location gl
-
-  def google_domain: -> GoogleBuilder
-  alias domain google_domain
-
-  def location: -> GoogleBuilder
-
-  def uule: -> GoogleBuilder
-
-  def lr: -> GoogleBuilder
-  alias language_restrict lr
-
-  def cr: -> GoogleBuilder
-  alias country_restrict cr
-
-  def safe_search: -> GoogleBuilder
-
-  def enable_nfpr: -> GoogleBuilder
-
-  def disable_filter: -> GoogleBuilder
-
-  def start: (Integer) -> Hash[String, String]
-
-  def next: -> Hash[String, String]
-
-  def params: -> GoogleBuilder
-
-  def scrapedo_url: -> URI::Generic
-
-  private
-
-  def load_yaml: (String) -> Hash[Object, String]
-
-  def find_config_source: (Symbol | String) -> [Symbol, Hash[Symbol, Object]]?
-
-  def blank?: (Object) -> bool
-
-end