webscraping_ai 4.0.1 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e312f29ac8666f33be3757599d08f062908be2a1a815973cc591d2cce44a4f4e
-  data.tar.gz: 6cb641d229bdf754fba1ffb2f717bc40db8bcad50158d6fc960d3d6de7448480
+  metadata.gz: cd7230f48fdc600563c0e7fe84ba65411f6363007854a9bdaae6b2ee6bfbf17d
+  data.tar.gz: a815bbc4645cd9eb23aba4e9cdafff0ac1c420f3a230580e002a59f7f19b28f6
 SHA512:
-  metadata.gz: 8d6838f0b67bf4426e2b1c6f48d5047fe4413c3d1e342e576aedc4a8df755f2fe4b591ccf747e8df955690d66ebe3ebf5b95747cc93794fa8fe0d90572050d19
-  data.tar.gz: 4d3966a1b3e7d4bd2b4a51cb1011f991ba43fc21f760fcce3496427af68256a5d2e45587089fc95e726f400577078935f83a178abd18c864f06da9312d29908e
+  metadata.gz: 8266c1f95d1432a37f817e3308e1c99c1795fba17fc105c4ef703561e89f84104f0d2734fda99dded525000baecd346d00c2ac3f0b838e251807d3651d20fae6
+  data.tar.gz: 6d075d815694065fde5b830d577100c8c45c9943798689ecc50b9959120d0486b10ede96b77344fb0803b8094f2855cf60d72237161b07a480e31b4531c62b1c

data/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 All notable changes to this project will be documented in this file. This project follows [Semantic Versioning](https://semver.org/).
 
+## 4.1.0 — 2026-09-25
+
+### Added
+
+- `Client#serp(q:, engine: nil, gl: nil, hl: nil, page: nil)` for the new `GET /serp` endpoint — parsed Google search results (organic results, related searches, pagination) as a Hash. Flat 15 credits per search. Raises `ArgumentError` when `q` is blank or not a String, or when `page` is not an Integer >= 1 (the server also rejects it with a 400, not billed; checking client-side saves the round trip). Pages are 1–100: the server rejects a `page` above 100 with a 400.
+- `bin/smoke.rb` live smoke script: asserts result shapes (not just the absence of exceptions), runs page tools with `js: false` on datacenter proxies (~32 credits per sweep), and redacts the API key from failure output.
+
+### Fixed
+
+- `Client#inspect` and `Configuration#inspect` no longer print the API key; it is shown as `api_key="[FILTERED]"`.
+
 ## 4.0.1 — 2026-07-17
 
 ### Changed

data/README.md

@@ -13,7 +13,7 @@ structured field extraction on any page. See the
 
 ```ruby
 # Gemfile
-gem "webscraping_ai", "~> 4.0"
+gem "webscraping_ai", "~> 4.1"
 ```
 
 Or:
@@ -60,6 +60,10 @@ data = client.fields(
   }
 )
 
+# Google search results (SERP) for a query
+results = client.serp(q: "coffee machines", gl: "us", hl: "en", page: 1)
+results["organic_results"].first["link"]
+
 # Check your account quota
 info = client.account
 # => { "remaining_api_calls" => 200_000, "resets_at" => 1_617_073_667, "remaining_concurrency" => 100 }
@@ -121,6 +125,34 @@ Endpoint-specific options:
 
 Returns: `String` for HTML/text responses, `Hash`/`Array` for JSON responses.
 
+### SERP (`#serp`)
+
+`#serp` is query-shaped rather than URL-shaped, so none of the page-fetch options above apply.
+It returns the parsed search results as a `Hash`. Flat 15 credits per search; failed searches are not charged.
+
+| Option | Type | Default | Description |
+| --- | --- | --- | --- |
+| `q` | `String` | — | Search query (required; blank or non-String raises `ArgumentError`) |
+| `engine` | `String` | `"google"` | Search engine; currently only `google` |
+| `gl` | `String` | `"us"` | Two-letter country code for the search |
+| `hl` | `String` | `"en"` | Two-letter language code for the results |
+| `page` | `Integer` | `1` | Results page number (10 results per page). Must be an `Integer` >= 1, otherwise `ArgumentError`; the server rejects values above 100 with a 400 (not billed) |
+
+```ruby
+results = client.serp(q: "coffee machines", gl: "gb", page: 2)
+results["search_information"]["organic_results_state"] # => "Results for exact spelling"
+results["organic_results"].each do |r|
+  puts "#{r["position"]}. #{r["title"]} — #{r["link"]}"
+end
+results["pagination"] # => { "current" => 2, "next" => 3 }
+```
+
+Response keys: `search_parameters` (`engine`, `q`, `gl`, `hl`, `page`), `search_information`
+(`query_displayed`, `organic_results_state`, optional `showing_results_for` and `total_results`),
+`organic_results` (`position` — 1-based within the page — `title`, `link`, `domain`, `displayed_link`,
+optional `snippet` and `date`), optional `related_searches` (`query`), and `pagination` (`current`, optional `next`).
+Optional keys are absent when Google does not show them.
+
 ## Error handling
 
 All API errors inherit from `WebScrapingAI::ApiError` and expose `#status`, `#message`, `#status_code`, `#status_message`, `#body`, and `#response_body`.
@@ -157,6 +189,17 @@ bundle exec rspec
 bundle exec rubocop
 ```
 
+## Smoke testing
+
+`bin/smoke.rb` hits every endpoint once against the live API, loading the gem from `lib/` so it tests the working tree. It is not part of the spec suite and costs ~32 credits per run: the four page calls run with `js: false` and `proxy: "datacenter"` (1 credit each), `question` and `fields` cost 6 each, and the SERP call is 15. Each case checks the result shape as well as exceptions (e.g. SERP must return organic results for the right query, `selected_multiple` must match something), and failure messages redact the API key.
+
+```bash
+WEBSCRAPING_AI_API_KEY=... bundle exec rake smoke
+# or: WEBSCRAPING_AI_API_KEY=... ruby bin/smoke.rb
+```
+
+Each endpoint prints an `ok` or `FAIL` line; the script exits non-zero if any call fails.
+
 ## Links
 
 - [WebScraping.AI](https://webscraping.ai) — features, pricing, signup

data/lib/webscraping_ai/client.rb

@@ -8,6 +8,7 @@ module WebScrapingAI
     DEVICES = %w[desktop mobile tablet].freeze
     TEXT_FORMATS = %w[plain xml json].freeze
     FORMATS = %w[json text].freeze
+    SERP_ENGINES = %w[google].freeze
 
     PAGE_FETCH_OPTIONS = %i[
       headers timeout js js_timeout wait_for proxy country
@@ -66,6 +67,23 @@ module WebScrapingAI
       get("/selected-multiple", url: url, selectors: Array(selectors), **opts.slice(*PAGE_FETCH_OPTIONS))
     end
 
+    # GET /serp — returns parsed search engine results for `q` as a Hash
+    # (search_parameters, search_information, organic_results, related_searches, pagination).
+    # Query-shaped: none of the page-fetch options apply. Flat 15 credits per search.
+    # `q` must be a non-blank String (sent as-is, untrimmed). `page`, when given, must be an
+    # Integer >= 1; the server rejects values above 100 with a 400 (not billed).
+    def serp(q:, engine: nil, gl: nil, hl: nil, page: nil)
+      raise ArgumentError, "q is required" if q.nil? || (q.is_a?(String) && q.strip.empty?)
+      raise ArgumentError, "q must be a String" unless q.is_a?(String)
+      raise ArgumentError, "page must be an Integer >= 1" unless page.nil? || (page.is_a?(Integer) && page >= 1)
+
+      get("/serp", q: q, engine: engine, gl: gl, hl: hl, page: page)
+    end
+
+    def inspect
+      "#<#{self.class.name} base_url=#{configuration.base_url.inspect} api_key=\"[FILTERED]\">"
+    end
+
     # GET /account — returns Hash with remaining_api_calls, resets_at, remaining_concurrency, email.
     def account
       get("/account")

data/lib/webscraping_ai/configuration.rb

@@ -14,5 +14,11 @@ module WebScrapingAI
       @adapter = nil
       @user_agent = "webscraping_ai-ruby/#{WebScrapingAI::VERSION}"
     end
+
+    def inspect
+      filtered = api_key.nil? ? "nil" : "\"[FILTERED]\""
+      "#<#{self.class.name} api_key=#{filtered} base_url=#{base_url.inspect} timeout=#{timeout.inspect} " \
+        "open_timeout=#{open_timeout.inspect} adapter=#{adapter.inspect} user_agent=#{user_agent.inspect}>"
+    end
   end
 end

data/lib/webscraping_ai/version.rb

@@ -1,3 +1,3 @@
 module WebScrapingAI
-  VERSION = "4.0.1".freeze
+  VERSION = "4.1.0".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webscraping_ai
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 4.1.0
 platform: ruby
 authors:
 - WebScraping.AI
@@ -66,7 +66,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Ruby client for the WebScraping.AI API.
 test_files: []