webscraping_ai 4.0.1 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e312f29ac8666f33be3757599d08f062908be2a1a815973cc591d2cce44a4f4e
-  data.tar.gz: 6cb641d229bdf754fba1ffb2f717bc40db8bcad50158d6fc960d3d6de7448480
+  metadata.gz: 12a9ee8045725e182d8c4600d17b6c2cdfb74264ae23b378ab59818d279f9e73
+  data.tar.gz: 8d26e529421132c3b615cf1b795e11546308f9e00c3e125c3e8d0f4d57b3c4af
 SHA512:
-  metadata.gz: 8d6838f0b67bf4426e2b1c6f48d5047fe4413c3d1e342e576aedc4a8df755f2fe4b591ccf747e8df955690d66ebe3ebf5b95747cc93794fa8fe0d90572050d19
-  data.tar.gz: 4d3966a1b3e7d4bd2b4a51cb1011f991ba43fc21f760fcce3496427af68256a5d2e45587089fc95e726f400577078935f83a178abd18c864f06da9312d29908e
+  metadata.gz: 28552acbb15d4b656f232a30e0476073ba55be3334b5752c87be65f900d950a7892491a907acc5f61cb90276cd4126db950a2555109d6df5138f948b5e7afb71
+  data.tar.gz: 3d8e917eb2ddba519649b89b3db73c2224a3185bf27946978120f5efe9778cc0685e7a43cce256434c3ae296ddc7418aa05d6ec1bebddde24fb7f292325c83e4

data/CHANGELOG.md

@@ -2,6 +2,27 @@
 
 All notable changes to this project will be documented in this file. This project follows [Semantic Versioning](https://semver.org/).
 
+## 4.2.0 — 2026-09-25
+### Added
+
+- `Client#data(url, country: nil, transcript: nil, transcript_language: nil, **params)` for the `GET /data` endpoint — structured JSON for a page on a supported site (e.g. YouTube, TikTok, X, LinkedIn, Instagram, Reddit) as a Hash with `request_parameters`, `parse_status` and `data`. Flat 15 credits per request. The client does not check the URL's site: new sites are added server-side, and an unsupported URL or page type returns a 400 that is not charged (`BadRequestError`); its message lists what is supported. Raises `ArgumentError` when `url` is blank or not a String. Extra keyword arguments are sent as-is as query params (String, Integer, Float or boolean; Floats are sent as plain decimals). `api_key`, `url`, `country`, `transcript` and `transcript_language` as extra params raise `ArgumentError` (use the named options).
+- `bin/smoke.rb` checks `/data` on a YouTube video and that `https://example.com/` gets the server's 400 "Unsupported URL" error (~47 credits per sweep).
+
+### Fixed
+
+- `TimeoutError` and `ConnectionError` redact `api_key=...` from the transport error's message and no longer carry the Faraday exception as `cause`, since it may include the request URL.
+
+## 4.1.0 — 2026-09-25
+
+### Added
+
+- `Client#serp(q:, engine: nil, gl: nil, hl: nil, page: nil)` for the new `GET /serp` endpoint — parsed Google search results (organic results, related searches, pagination) as a Hash. Flat 15 credits per search. Raises `ArgumentError` when `q` is blank or not a String, or when `page` is not an Integer >= 1 (the server also rejects it with a 400, not billed; checking client-side saves the round trip). Pages are 1–100: the server rejects a `page` above 100 with a 400.
+- `bin/smoke.rb` live smoke script: asserts result shapes (not just the absence of exceptions), runs page tools with `js: false` on datacenter proxies (~32 credits per sweep), and redacts the API key from failure output.
+
+### Fixed
+
+- `Client#inspect` and `Configuration#inspect` no longer print the API key; it is shown as `api_key="[FILTERED]"`.
+
 ## 4.0.1 — 2026-07-17
 
 ### Changed

data/README.md

@@ -13,7 +13,7 @@ structured field extraction on any page. See the
 
 ```ruby
 # Gemfile
-gem "webscraping_ai", "~> 4.0"
+gem "webscraping_ai", "~> 4.1"
 ```
 
 Or:
@@ -60,6 +60,14 @@ data = client.fields(
   }
 )
 
+# Google search results (SERP) for a query
+results = client.serp(q: "coffee machines", gl: "us", hl: "en", page: 1)
+results["organic_results"].first["link"]
+
+# Structured data for a page on a supported site (YouTube, TikTok, X, LinkedIn, Instagram, Reddit, ...)
+video = client.data("https://www.youtube.com/watch?v=dQw4w9WgXcQ")
+video["data"]["title"]
+
 # Check your account quota
 info = client.account
 # => { "remaining_api_calls" => 200_000, "resets_at" => 1_617_073_667, "remaining_concurrency" => 100 }
@@ -121,6 +129,72 @@ Endpoint-specific options:
 
 Returns: `String` for HTML/text responses, `Hash`/`Array` for JSON responses.
 
+### SERP (`#serp`)
+
+`#serp` is query-shaped rather than URL-shaped, so none of the page-fetch options above apply.
+It returns the parsed search results as a `Hash`. Flat 15 credits per search; failed searches are not charged.
+
+| Option | Type | Default | Description |
+| --- | --- | --- | --- |
+| `q` | `String` | — | Search query (required; blank or non-String raises `ArgumentError`) |
+| `engine` | `String` | `"google"` | Search engine; currently only `google` |
+| `gl` | `String` | `"us"` | Two-letter country code for the search |
+| `hl` | `String` | `"en"` | Two-letter language code for the results |
+| `page` | `Integer` | `1` | Results page number (10 results per page). Must be an `Integer` >= 1, otherwise `ArgumentError`; the server rejects values above 100 with a 400 (not billed) |
+
+```ruby
+results = client.serp(q: "coffee machines", gl: "gb", page: 2)
+results["search_information"]["organic_results_state"] # => "Results for exact spelling"
+results["organic_results"].each do |r|
+  puts "#{r["position"]}. #{r["title"]} — #{r["link"]}"
+end
+results["pagination"] # => { "current" => 2, "next" => 3 }
+```
+
+Response keys: `search_parameters` (`engine`, `q`, `gl`, `hl`, `page`), `search_information`
+(`query_displayed`, `organic_results_state`, optional `showing_results_for` and `total_results`),
+`organic_results` (`position` — 1-based within the page — `title`, `link`, `domain`, `displayed_link`,
+optional `snippet` and `date`), optional `related_searches` (`query`), and `pagination` (`current`, optional `next`).
+Optional keys are absent when Google does not show them.
+
+### Structured data (`#data`)
+
+`#data(url, country: nil, transcript: nil, transcript_language: nil, **params)` returns structured JSON
+for a public page on a supported site as a `Hash`. Pass the page's normal URL; the site (`provider`) and
+page kind (`type`) are detected from it. Flat 15 credits per request, including pages that parse empty
+(`parse_status` `"parse_failed"`) or no longer exist (`"not_found"`); failed fetches are not charged.
+None of the page-fetch options above apply.
+
+Supported sites today include, for example, YouTube (video/channel/playlist), TikTok (video/profile),
+X/Twitter (tweet/profile), LinkedIn (company/job/profile), Instagram (post/reel/profile) and Reddit
+(post/subreddit/user). **More sites are added server-side**, and they work with this gem without an
+upgrade: the client never checks the URL's site. An unsupported URL or page type returns a 400 that is
+not charged (`WebScrapingAI::BadRequestError`). Its message lists what is supported.
+
+| Option | Type | Default | Description |
+| --- | --- | --- | --- |
+| `url` | `String` | — | Page URL (required, positional; blank or non-String raises `ArgumentError`) |
+| `country` | `String` | `"us"` | Two-letter country code of the proxy used to fetch the page, `us` by default |
+| `transcript` | `Boolean` | `false` | YouTube videos only. Also fetch the video's transcript into `data.transcript`. It's null when no matching captions are available. If the transcript fetch itself fails, the whole request fails with a 500 and is not charged |
+| `transcript_language` | `String` | — | Caption language to pick, e.g. `en` or `de`. Without it, English is preferred, then the first available track. If the video has no captions in that language, `data.transcript` is null |
+| `**params` | `String`, `Integer`, `Float`, boolean | — | Extra query params sent as-is, for provider-specific params added later (`nil` omits one). `api_key`, `url`, `country`, `transcript` and `transcript_language` raise `ArgumentError`; use the named options for the last three |
+
+```ruby
+result = client.data("https://www.youtube.com/watch?v=dQw4w9WgXcQ", transcript: true)
+result["request_parameters"] # => { "url" => "...", "provider" => "youtube", "type" => "video" }
+result["parse_status"]       # => "ok" (or "parse_failed" / "not_found")
+result["data"]["title"]      # shape depends on provider and type; may be nil
+
+begin
+  client.data("https://example.com/")
+rescue WebScrapingAI::BadRequestError => e
+  e.message # => "Unsupported URL for /data. Supported sites: youtube, tiktok, ..."
+end
+```
+
+`provider`, `type` and `parse_status` are open sets of strings, and `data` is the decoded JSON as-is
+(no per-site classes), so new sites and fields show up without a gem release.
+
 ## Error handling
 
 All API errors inherit from `WebScrapingAI::ApiError` and expose `#status`, `#message`, `#status_code`, `#status_message`, `#body`, and `#response_body`.
@@ -157,6 +231,17 @@ bundle exec rspec
 bundle exec rubocop
 ```
 
+## Smoke testing
+
+`bin/smoke.rb` hits every endpoint once against the live API, loading the gem from `lib/` so it tests the working tree. It is not part of the spec suite and costs ~47 credits per run: the four page calls run with `js: false` and `proxy: "datacenter"` (1 credit each), `question` and `fields` cost 6 each, and the SERP and `/data` (YouTube video) calls are 15 each. A second `/data` call on `https://example.com/` must come back as the server's free 400, proving there is no client-side site filter. Each case checks the result shape as well as exceptions (e.g. SERP must return organic results for the right query, `/data` must parse a title, `selected_multiple` must match something), and failure messages redact the API key.
+
+```bash
+WEBSCRAPING_AI_API_KEY=... bundle exec rake smoke
+# or: WEBSCRAPING_AI_API_KEY=... ruby bin/smoke.rb
+```
+
+Each endpoint prints an `ok` or `FAIL` line; the script exits non-zero if any call fails.
+
 ## Links
 
 - [WebScraping.AI](https://webscraping.ai) — features, pricing, signup

data/lib/webscraping_ai/client.rb

@@ -8,6 +8,7 @@ module WebScrapingAI
     DEVICES = %w[desktop mobile tablet].freeze
     TEXT_FORMATS = %w[plain xml json].freeze
     FORMATS = %w[json text].freeze
+    SERP_ENGINES = %w[google].freeze
 
     PAGE_FETCH_OPTIONS = %i[
       headers timeout js js_timeout wait_for proxy country
@@ -66,6 +67,51 @@ module WebScrapingAI
       get("/selected-multiple", url: url, selectors: Array(selectors), **opts.slice(*PAGE_FETCH_OPTIONS))
     end
 
+    # GET /serp — returns parsed search engine results for `q` as a Hash
+    # (search_parameters, search_information, organic_results, related_searches, pagination).
+    # Query-shaped: none of the page-fetch options apply. Flat 15 credits per search.
+    # `q` must be a non-blank String (sent as-is, untrimmed). `page`, when given, must be an
+    # Integer >= 1; the server rejects values above 100 with a 400 (not billed).
+    def serp(q:, engine: nil, gl: nil, hl: nil, page: nil)
+      raise ArgumentError, "q is required" if q.nil? || (q.is_a?(String) && q.strip.empty?)
+      raise ArgumentError, "q must be a String" unless q.is_a?(String)
+      raise ArgumentError, "page must be an Integer >= 1" unless page.nil? || (page.is_a?(Integer) && page >= 1)
+
+      get("/serp", q: q, engine: engine, gl: gl, hl: hl, page: page)
+    end
+
+    # GET /data — returns structured JSON for a page on a supported site as a Hash
+    # (request_parameters: url/provider/type, parse_status, data). Flat 15 credits per request.
+    #
+    # The client never checks which site `url` belongs to: supported sites (e.g. YouTube, TikTok,
+    # X/Twitter, LinkedIn, Instagram, Reddit) are added server-side. An unsupported URL or page type
+    # returns a 400 that is not charged (BadRequestError); its message lists what is supported.
+    # `provider`, `type` and `parse_status` are open sets of strings; `data` may be nil.
+    #
+    # `country`: two-letter country code of the proxy used to fetch the page, `us` by default.
+    # `transcript`: YouTube videos only. Also fetch the video's transcript into `data.transcript`.
+    # It's null when no matching captions are available. If the transcript fetch itself fails, the
+    # whole request fails with a 500 and is not charged.
+    # `transcript_language`: caption language to pick, e.g. `en` or `de`. Without it, English is
+    # preferred, then the first available track. If the video has no captions in that language,
+    # `data.transcript` is null.
+    #
+    # Any other keyword arguments are sent as-is as extra query params (for provider-specific params
+    # added later). Values must be String, Integer, Float or boolean (nil omits the param); the names
+    # `api_key`, `url`, `country`, `transcript` and `transcript_language` raise ArgumentError.
+    # None of the page-fetch options apply.
+    def data(url, country: nil, transcript: nil, transcript_language: nil, **params)
+      raise ArgumentError, "url is required" if url.nil? || (url.is_a?(String) && url.strip.empty?)
+      raise ArgumentError, "url must be a String" unless url.is_a?(String)
+
+      named = { country: country, transcript: transcript, transcript_language: transcript_language }.compact
+      get("/data", url: url, **data_extra_params(params), **named)
+    end
+
+    def inspect
+      "#<#{self.class.name} base_url=#{configuration.base_url.inspect} api_key=\"[FILTERED]\">"
+    end
+
     # GET /account — returns Hash with remaining_api_calls, resets_at, remaining_concurrency, email.
     def account
       get("/account")
@@ -73,6 +119,54 @@ module WebScrapingAI
 
     private
 
+    DATA_RESERVED_PARAMS = %w[api_key url].freeze
+    DATA_TYPED_PARAMS = %w[country transcript transcript_language].freeze
+    private_constant :DATA_RESERVED_PARAMS, :DATA_TYPED_PARAMS
+
+    # Extra /data query params are passed through untouched (same encoder, so `&`/`=` are escaped),
+    # except that they can't override the credentials or the target URL, and can't repeat a typed
+    # param (a string key like "country" would otherwise collide with `country:`).
+    def data_extra_params(params)
+      params.each_with_object({}) do |(key, value), extra|
+        name = key.to_s
+        raise ArgumentError, "#{name} can't be passed as an extra /data param" if DATA_RESERVED_PARAMS.include?(name)
+        if DATA_TYPED_PARAMS.include?(name)
+          raise ArgumentError, "#{name} can't be passed as an extra /data param; use the #{name}: option"
+        end
+
+        extra[name.to_sym] = data_extra_value(name, value)
+      end
+    end
+
+    def data_extra_value(name, value)
+      case value
+      when nil, String, Integer, true, false then value
+      when Float
+        raise ArgumentError, "extra /data param #{name} must be a finite number" unless value.finite?
+
+        plain_float(value)
+      else
+        raise ArgumentError, "extra /data param #{name} must be a String, Integer, Float or boolean"
+      end
+    end
+
+    # Float#to_s switches to exponent notation (1.0e+20, 1.5e-07); send plain decimals instead.
+    def plain_float(value)
+      text = value.to_s
+      return text unless text.include?("e")
+
+      mantissa, exponent = text.split("e")
+      fraction_digits = mantissa.split(".")[1].to_s.sub(/0+\z/, "").length
+      format("%.#{[fraction_digits - exponent.to_i, 0].max}f", value)
+    end
+
+    def redact(text)
+      key = configuration.api_key.to_s
+      text = text.to_s
+      text = text.gsub(key, "[FILTERED]") unless key.empty?
+      text.gsub(/api_key=[^&\s"']*/, "api_key=[FILTERED]")
+    end
+
     def connection
       @connection ||= Faraday.new(url: configuration.base_url) do |conn|
         conn.options.timeout = configuration.timeout
@@ -84,15 +178,17 @@ module WebScrapingAI
       end
     end
 
+    # Transport errors are re-raised with `cause: nil` and a redacted message: the Faraday error (and
+    # anything it wraps) may carry the request URL, which contains the API key.
     def get(path, **params)
       response = connection.get(path) do |req|
         req.params = params.merge(api_key: configuration.api_key)
       end
       handle_response(response)
     rescue Faraday::TimeoutError => e
-      raise TimeoutError, e.message
+      raise TimeoutError.new(redact(e.message)), cause: nil
     rescue Faraday::ConnectionFailed => e
-      raise ConnectionError, e.message
+      raise ConnectionError.new(redact(e.message)), cause: nil
     end
 
     def handle_response(response)

data/lib/webscraping_ai/configuration.rb

@@ -14,5 +14,11 @@ module WebScrapingAI
       @adapter = nil
       @user_agent = "webscraping_ai-ruby/#{WebScrapingAI::VERSION}"
     end
+
+    def inspect
+      filtered = api_key.nil? ? "nil" : "\"[FILTERED]\""
+      "#<#{self.class.name} api_key=#{filtered} base_url=#{base_url.inspect} timeout=#{timeout.inspect} " \
+        "open_timeout=#{open_timeout.inspect} adapter=#{adapter.inspect} user_agent=#{user_agent.inspect}>"
+    end
   end
 end

data/lib/webscraping_ai/version.rb

@@ -1,3 +1,3 @@
 module WebScrapingAI
-  VERSION = "4.0.1".freeze
+  VERSION = "4.2.0".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webscraping_ai
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 4.2.0
 platform: ruby
 authors:
 - WebScraping.AI
@@ -66,7 +66,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Ruby client for the WebScraping.AI API.
 test_files: []