scrapio 1.2.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 31b0030dedce45ac96d108f4869e218f6b40ea6db264f981319a252610366961
-  data.tar.gz: 79d78805bb3ec04fe89dacdc179b775397f02bcef4c0eb58c724905fa0f1117c
+  metadata.gz: a9bb182980da7baea813ef5f1763220cb8524a027215ca7165dd69fed53d0341
+  data.tar.gz: 8eafab5f4f8ae2fa05c7a7ff35911e8bf5ffc99578ef841cb580cde5061881a2
 SHA512:
-  metadata.gz: d43cf4d5884ce7e8d26ec17e047bc1b60f472f212eb1dacd592ccef58b30ecb3a0adc08b5bb33fe0272942bad6f19ec523d1217312b2995375c9f3eaa37bd940
-  data.tar.gz: 5c0c8c7425d7a27629fe9c3db0ef473de69f64dd7267a98d193a31614f350cf45c81fa26a961972b7e2daae14d70b0328a58e6828657b348b434523b3d133cd0
+  metadata.gz: 0de0b77f446621dccd8256ed12b900024d3509e19de295e4b283443b7123ac505cf3c3ca391cd05255935d65e2312b59741190c43ba2b387269e7064738709ce
+  data.tar.gz: c0ea3e7e07234cdfe05ca9eddd70cfec7c1b6bd4efa1e6c40c4bac3c36b1e36e2fdad5d1d6a12d7f9d6da5cca83ce1dba341b4d87264b1c5778802d8ca7abae3

data/lib/scrapio/client.rb

@@ -13,11 +13,20 @@ require_relative "resources/map"
 require_relative "resources/booking"
 require_relative "resources/agoda"
 require_relative "resources/airbnb"
+require_relative "resources/autotrader"
+require_relative "resources/glassdoor"
+require_relative "resources/trustpilot"
+require_relative "resources/ziprecruiter"
+require_relative "resources/dice"
+require_relative "resources/simplyhired"
+require_relative "resources/builtin"
+require_relative "resources/wellfound"
 require_relative "resources/chatgpt"
 require_relative "resources/perplexity"
 require_relative "resources/gemini"
 require_relative "resources/bing"
 require_relative "resources/reddit"
+require_relative "resources/hacker_news"
 require_relative "resources/tiktok"
 require_relative "resources/target"
 require_relative "resources/apple_app_store"
@@ -26,7 +35,8 @@ module Scrapio
   class Client
     attr_reader :fetch, :google, :amazon, :walmart, :youtube, :jobs, :crawl, :interact,
                 :fast_search, :search, :map, :booking, :agoda,
-                :airbnb, :chatgpt, :perplexity, :gemini, :bing, :reddit, :tiktok, :target, :apple_app_store
+                :airbnb, :autotrader, :chatgpt, :perplexity, :gemini, :bing, :reddit, :tiktok, :target, :apple_app_store,
+                :hacker_news, :glassdoor, :trustpilot, :ziprecruiter, :dice, :simplyhired, :builtin, :wellfound
 
     def initialize(api_key, base_url: HttpClient::DEFAULT_BASE_URL, timeout: HttpClient::DEFAULT_TIMEOUT)
       http = HttpClient.new(api_key, base_url: base_url, timeout: timeout)
@@ -45,6 +55,14 @@ module Scrapio
       @booking         = Resources::Booking.new(http)
       @agoda           = Resources::Agoda.new(http)
       @airbnb          = Resources::Airbnb.new(http)
+      @autotrader      = Resources::Autotrader.new(http)
+      @glassdoor       = Resources::Glassdoor.new(http)
+      @trustpilot      = Resources::Trustpilot.new(http)
+      @ziprecruiter    = Resources::Ziprecruiter.new(http)
+      @dice            = Resources::Dice.new(http)
+      @simplyhired     = Resources::Simplyhired.new(http)
+      @builtin         = Resources::Builtin.new(http)
+      @wellfound       = Resources::Wellfound.new(http)
       @chatgpt         = Resources::Chatgpt.new(http)
       @perplexity      = Resources::Perplexity.new(http)
       @gemini          = Resources::Gemini.new(http)
@@ -53,6 +71,7 @@ module Scrapio
       @tiktok          = Resources::TikTok.new(http)
       @target          = Resources::Target.new(http)
       @apple_app_store = Resources::AppleAppStore.new(http)
+      @hacker_news     = Resources::HackerNews.new(http)
     end
   end
 end

data/lib/scrapio/proxy_mode.rb

@@ -0,0 +1,23 @@
+module Scrapio
+  # How far a request may escalate to get past anti-bot defenses. Pass one of
+  # these as `proxy:` to Fetch#fetch, Crawl#crawl, Interact#interact, or as the
+  # "proxy" key of a Map#execute request.
+  #
+  #   BASIC    no proxy at all (cheapest). If the target blocks it the request
+  #            fails with `proxy_required`; retry with AUTO.
+  #   STEALTH  straight to the strongest tier, a hardened browser behind a
+  #            residential proxy (paid plans).
+  #   AUTO     cheap steps first, escalating to a residential proxy only when
+  #            needed, billed by the step that served the request (paid plans, the
+  #            default).
+  #
+  # The Free plan is always BASIC; STEALTH and AUTO return 402 there. The response's
+  # `usage` hash reports what the request cost: credits, base, surcharge and tier
+  # ("standard" +0, "proxy" +4, "proxy_browser" +10, "stealth" +65, "cloudflare" +65).
+  module ProxyMode
+    BASIC = "basic".freeze
+    STEALTH = "stealth".freeze
+    AUTO = "auto".freeze
+    ALL = [BASIC, STEALTH, AUTO].freeze
+  end
+end

data/lib/scrapio/resources/autotrader.rb

@@ -0,0 +1,21 @@
+module Scrapio
+  module Resources
+    class Autotrader
+      def initialize(http) = @http = http
+
+      def search(postcode:, radius: nil, make: nil, model: nil, year_from: nil, year_to: nil, exclude_writeoffs: nil, page: nil, geo: nil)
+        @http.get("/v1/autotrader/search", {
+          postcode: postcode, radius: radius, make: make, model: model,
+          year_from: year_from, year_to: year_to,
+          exclude_writeoffs: exclude_writeoffs, page: page, geo: geo,
+        })
+      end
+
+      def get_listing(listing_id, geo: nil)
+        @http.get("/v1/autotrader/listing", {
+          listing_id: listing_id, geo: geo,
+        })
+      end
+    end
+  end
+end

data/lib/scrapio/resources/builtin.rb

@@ -0,0 +1,13 @@
+module Scrapio
+  module Resources
+    class Builtin
+      def initialize(http) = @http = http
+
+      def jobs(query:, limit: nil, cursor: nil)
+        @http.get("/v1/builtin/jobs", {
+          query: query, limit: limit, cursor: cursor,
+        })
+      end
+    end
+  end
+end

data/lib/scrapio/resources/crawl.rb

@@ -3,12 +3,13 @@ module Scrapio
     class Crawl
       def initialize(http) = @http = http
 
+      # proxy: "basic", "stealth" or "auto" (see Scrapio::ProxyMode); applies to every page.
       def crawl(seeds:, max_pages: nil, max_depth: nil, same_domain_only: nil,
-                output: nil, extract: nil, timeout_ms: nil)
+                output: nil, extract: nil, timeout_ms: nil, proxy: nil)
         @http.post("/v1/crawl", {
           seeds: seeds, max_pages: max_pages, max_depth: max_depth,
           same_domain_only: same_domain_only, output: output,
-          extract: extract, timeout_ms: timeout_ms,
+          extract: extract, timeout_ms: timeout_ms, proxy: proxy,
         })
       end
     end

data/lib/scrapio/resources/dice.rb

@@ -0,0 +1,13 @@
+module Scrapio
+  module Resources
+    class Dice
+      def initialize(http) = @http = http
+
+      def jobs(query:, location: nil, limit: nil, cursor: nil)
+        @http.get("/v1/dice/jobs", {
+          query: query, location: location, limit: limit, cursor: cursor,
+        })
+      end
+    end
+  end
+end

data/lib/scrapio/resources/fetch.rb

@@ -3,11 +3,15 @@ module Scrapio
     class Fetch
       def initialize(http) = @http = http
 
+      # proxy: "basic", "stealth" or "auto" (see Scrapio::ProxyMode); nil uses the plan
+      # default. timeout_ms is the per-attempt budget: 10000 by default, 20000 with
+      # proxy: "stealth".
       def fetch(url:, render_js: nil, device: nil, session: nil, output: nil,
-                extract: nil, wait_for: nil, timeout_ms: nil)
+                extract: nil, wait_for: nil, timeout_ms: nil, proxy: nil)
         @http.post("/v1/fetch", {
           url: url, render_js: render_js, device: device, session: session,
           output: output, extract: extract, wait_for: wait_for, timeout_ms: timeout_ms,
+          proxy: proxy,
         })
       end
     end

data/lib/scrapio/resources/glassdoor.rb

@@ -0,0 +1,17 @@
+module Scrapio
+  module Resources
+    class Glassdoor
+      def initialize(http) = @http = http
+
+      def jobs(query:, location: nil, limit: nil, cursor: nil)
+        @http.get("/v1/glassdoor/jobs", {
+          query: query, location: location, limit: limit, cursor: cursor,
+        })
+      end
+
+      def job_detail(job_id:)
+        @http.get("/v1/glassdoor/job", { job_id: job_id })
+      end
+    end
+  end
+end

data/lib/scrapio/resources/hacker_news.rb

@@ -0,0 +1,30 @@
+module Scrapio
+  module Resources
+    class HackerNews
+      def initialize(http) = @http = http
+
+      def get_item(id:, include_comments: nil)
+        @http.get("/v1/hacker-news/item", { id: id, include_comments: include_comments })
+      end
+
+      def get_user(username:)
+        @http.get("/v1/hacker-news/user", { username: username })
+      end
+
+      def search(query:, tags: nil, since: nil, sort: nil, page: nil, hits_per_page: nil)
+        @http.get("/v1/hacker-news/search", {
+          query: query,
+          tags: tags && tags.join(","),
+          since: since,
+          sort: sort,
+          page: page,
+          hits_per_page: hits_per_page
+        })
+      end
+
+      def get_stories(type:, limit: nil)
+        @http.get("/v1/hacker-news/stories", { type: type, limit: limit })
+      end
+    end
+  end
+end

data/lib/scrapio/resources/interact.rb

@@ -3,11 +3,12 @@ module Scrapio
     class Interact
       def initialize(http) = @http = http
 
+      # proxy: "basic", "stealth" or "auto" (see Scrapio::ProxyMode); nil uses the plan default.
       def interact(url:, actions:, device: nil, session: nil,
-                   output: nil, extract: nil, timeout_ms: nil)
+                   output: nil, extract: nil, timeout_ms: nil, proxy: nil)
         @http.post("/v1/interact", {
           url: url, actions: actions, device: device, session: session,
-          output: output, extract: extract, timeout_ms: timeout_ms,
+          output: output, extract: extract, timeout_ms: timeout_ms, proxy: proxy,
         })
       end
     end

data/lib/scrapio/resources/map.rb

@@ -1,7 +1,8 @@
 module Scrapio
   module Resources
     # POST /v1/map -- site-map/URL-discovery workflow. Untyped request/response,
-    # same reasoning as Search.
+    # same reasoning as Search. The request accepts "proxy" ("basic", "stealth" or
+    # "auto", see Scrapio::ProxyMode).
     class Map
       def initialize(http) = @http = http
 

data/lib/scrapio/resources/simplyhired.rb

@@ -0,0 +1,13 @@
+module Scrapio
+  module Resources
+    class Simplyhired
+      def initialize(http) = @http = http
+
+      def jobs(query:, location: nil, limit: nil, cursor: nil)
+        @http.get("/v1/simplyhired/jobs", {
+          query: query, location: location, limit: limit, cursor: cursor,
+        })
+      end
+    end
+  end
+end

data/lib/scrapio/resources/trustpilot.rb

@@ -0,0 +1,14 @@
+module Scrapio
+  module Resources
+    class Trustpilot
+      def initialize(http) = @http = http
+
+      def reviews(domain:, stars: nil, language: nil, verified: nil, limit: nil, cursor: nil)
+        @http.get("/v1/trustpilot/reviews", {
+          domain: domain, stars: stars, language: language,
+          verified: verified, limit: limit, cursor: cursor,
+        })
+      end
+    end
+  end
+end

data/lib/scrapio/resources/wellfound.rb

@@ -0,0 +1,13 @@
+module Scrapio
+  module Resources
+    class Wellfound
+      def initialize(http) = @http = http
+
+      def jobs(role:, limit: nil, cursor: nil)
+        @http.get("/v1/wellfound/jobs", {
+          role: role, limit: limit, cursor: cursor,
+        })
+      end
+    end
+  end
+end

data/lib/scrapio/resources/ziprecruiter.rb

@@ -0,0 +1,13 @@
+module Scrapio
+  module Resources
+    class Ziprecruiter
+      def initialize(http) = @http = http
+
+      def jobs(query:, location: nil, limit: nil, cursor: nil)
+        @http.get("/v1/ziprecruiter/jobs", {
+          query: query, location: location, limit: limit, cursor: cursor,
+        })
+      end
+    end
+  end
+end

data/lib/scrapio.rb

@@ -1,4 +1,5 @@
 require_relative "scrapio/errors"
+require_relative "scrapio/proxy_mode"
 require_relative "scrapio/client"
 
 module Scrapio

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrapio
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Scrapio
@@ -22,28 +22,38 @@ files:
 - lib/scrapio/client.rb
 - lib/scrapio/errors.rb
 - lib/scrapio/http_client.rb
+- lib/scrapio/proxy_mode.rb
 - lib/scrapio/resources/agoda.rb
 - lib/scrapio/resources/airbnb.rb
 - lib/scrapio/resources/amazon.rb
 - lib/scrapio/resources/apple_app_store.rb
+- lib/scrapio/resources/autotrader.rb
 - lib/scrapio/resources/bing.rb
 - lib/scrapio/resources/booking.rb
+- lib/scrapio/resources/builtin.rb
 - lib/scrapio/resources/chatgpt.rb
 - lib/scrapio/resources/crawl.rb
+- lib/scrapio/resources/dice.rb
 - lib/scrapio/resources/fast_search.rb
 - lib/scrapio/resources/fetch.rb
 - lib/scrapio/resources/gemini.rb
+- lib/scrapio/resources/glassdoor.rb
 - lib/scrapio/resources/google.rb
+- lib/scrapio/resources/hacker_news.rb
 - lib/scrapio/resources/interact.rb
 - lib/scrapio/resources/jobs.rb
 - lib/scrapio/resources/map.rb
 - lib/scrapio/resources/perplexity.rb
 - lib/scrapio/resources/reddit.rb
 - lib/scrapio/resources/search.rb
+- lib/scrapio/resources/simplyhired.rb
 - lib/scrapio/resources/target.rb
 - lib/scrapio/resources/tiktok.rb
+- lib/scrapio/resources/trustpilot.rb
 - lib/scrapio/resources/walmart.rb
+- lib/scrapio/resources/wellfound.rb
 - lib/scrapio/resources/youtube.rb
+- lib/scrapio/resources/ziprecruiter.rb
 homepage: https://scrapio.dev
 licenses:
 - MIT