scrapio 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/scrapio/client.rb +20 -9
- data/lib/scrapio/resources/agoda.rb +34 -0
- data/lib/scrapio/resources/booking.rb +35 -0
- data/lib/scrapio/resources/fast_search.rb +11 -0
- data/lib/scrapio/resources/map.rb +13 -0
- data/lib/scrapio/resources/search.rb +15 -0
- data/lib/scrapio/resources/walmart.rb +5 -0
- metadata +10 -8
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0b4ceb44c4640ab93268a449b90b3c2cb5ee2c4ee11bf0035b1661765662d104
|
|
4
|
+
data.tar.gz: cad932294af50b9a90d88f3c6008645346e77f0814844d92b6911a7964d6c308
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 13270e660b4bb3b69265147a65c07bac5eb3ce03410e8fc260cb246631c703f98b6c261aabf547e81f7b8bebae0c0f4929cc0e085602ad4282c4aac637075176
|
|
7
|
+
data.tar.gz: 8d3d862efcafef5a430abad6f2da6b8fa01054951c0fbf11858b1de11a140773f0acf7359e20ae3459b2de85741f992668fbe1d35b7eb7cb8b5b3b2fc089245e
|
data/lib/scrapio/client.rb
CHANGED
|
@@ -7,22 +7,33 @@ require_relative "resources/youtube"
|
|
|
7
7
|
require_relative "resources/jobs"
|
|
8
8
|
require_relative "resources/crawl"
|
|
9
9
|
require_relative "resources/interact"
|
|
10
|
+
require_relative "resources/fast_search"
|
|
11
|
+
require_relative "resources/search"
|
|
12
|
+
require_relative "resources/map"
|
|
13
|
+
require_relative "resources/booking"
|
|
14
|
+
require_relative "resources/agoda"
|
|
10
15
|
|
|
11
16
|
module Scrapio
|
|
12
17
|
class Client
|
|
13
|
-
attr_reader :fetch, :google, :amazon, :walmart, :youtube, :jobs, :crawl, :interact
|
|
18
|
+
attr_reader :fetch, :google, :amazon, :walmart, :youtube, :jobs, :crawl, :interact,
|
|
19
|
+
:fast_search, :search, :map, :booking, :agoda
|
|
14
20
|
|
|
15
21
|
def initialize(api_key, base_url: HttpClient::DEFAULT_BASE_URL, timeout: HttpClient::DEFAULT_TIMEOUT)
|
|
16
22
|
http = HttpClient.new(api_key, base_url: base_url, timeout: timeout)
|
|
17
23
|
|
|
18
|
-
@fetch
|
|
19
|
-
@google
|
|
20
|
-
@amazon
|
|
21
|
-
@walmart
|
|
22
|
-
@youtube
|
|
23
|
-
@jobs
|
|
24
|
-
@crawl
|
|
25
|
-
@interact
|
|
24
|
+
@fetch = Resources::Fetch.new(http)
|
|
25
|
+
@google = Resources::Google.new(http)
|
|
26
|
+
@amazon = Resources::Amazon.new(http)
|
|
27
|
+
@walmart = Resources::Walmart.new(http)
|
|
28
|
+
@youtube = Resources::YouTube.new(http)
|
|
29
|
+
@jobs = Resources::Jobs.new(http)
|
|
30
|
+
@crawl = Resources::Crawl.new(http)
|
|
31
|
+
@interact = Resources::Interact.new(http)
|
|
32
|
+
@fast_search = Resources::FastSearch.new(http)
|
|
33
|
+
@search = Resources::Search.new(http)
|
|
34
|
+
@map = Resources::Map.new(http)
|
|
35
|
+
@booking = Resources::Booking.new(http)
|
|
36
|
+
@agoda = Resources::Agoda.new(http)
|
|
26
37
|
end
|
|
27
38
|
end
|
|
28
39
|
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
module Scrapio
|
|
2
|
+
module Resources
|
|
3
|
+
class Agoda
|
|
4
|
+
def initialize(http) = @http = http
|
|
5
|
+
|
|
6
|
+
def search(location:, check_in:, check_out:, adults: nil, children: nil, children_ages: nil,
|
|
7
|
+
rooms: nil, currency: nil, pos_country: nil, sort_by: nil, min_price: nil,
|
|
8
|
+
max_price: nil, stars: nil, min_review_score: nil, page: nil)
|
|
9
|
+
@http.get("/v1/agoda/search", {
|
|
10
|
+
location: location, check_in: check_in, check_out: check_out,
|
|
11
|
+
adults: adults, children: children, children_ages: children_ages, rooms: rooms,
|
|
12
|
+
currency: currency, pos_country: pos_country, sort_by: sort_by,
|
|
13
|
+
min_price: min_price, max_price: max_price, stars: stars,
|
|
14
|
+
min_review_score: min_review_score, page: page,
|
|
15
|
+
})
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Provide url, or property_id together with city_id (from a prior
|
|
19
|
+
# search's resolved_city_id).
|
|
20
|
+
def get_property(property_id: nil, city_id: nil, url: nil, check_in:, check_out:, adults: nil,
|
|
21
|
+
children: nil, children_ages: nil, rooms: nil, currency: nil, pos_country: nil)
|
|
22
|
+
@http.get("/v1/agoda/property", {
|
|
23
|
+
property_id: property_id, city_id: city_id, url: url, check_in: check_in, check_out: check_out,
|
|
24
|
+
adults: adults, children: children, children_ages: children_ages, rooms: rooms,
|
|
25
|
+
currency: currency, pos_country: pos_country,
|
|
26
|
+
})
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def get_reviews(property_id: nil, url: nil, page: nil, language: nil)
|
|
30
|
+
@http.get("/v1/agoda/reviews", { property_id: property_id, url: url, page: page, language: language })
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
module Scrapio
|
|
2
|
+
module Resources
|
|
3
|
+
class Booking
|
|
4
|
+
def initialize(http) = @http = http
|
|
5
|
+
|
|
6
|
+
def search(location:, check_in:, check_out:, adults: nil, children: nil, children_ages: nil,
|
|
7
|
+
rooms: nil, currency: nil, pos_country: nil, sort_by: nil, min_price: nil,
|
|
8
|
+
max_price: nil, stars: nil, min_review_score: nil, page: nil)
|
|
9
|
+
@http.get("/v1/booking/search", {
|
|
10
|
+
location: location, check_in: check_in, check_out: check_out,
|
|
11
|
+
adults: adults, children: children, children_ages: children_ages, rooms: rooms,
|
|
12
|
+
currency: currency, pos_country: pos_country, sort_by: sort_by,
|
|
13
|
+
min_price: min_price, max_price: max_price, stars: stars,
|
|
14
|
+
min_review_score: min_review_score, page: page,
|
|
15
|
+
})
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def get_property(property_id: nil, url: nil, check_in:, check_out:, adults: nil,
|
|
19
|
+
children: nil, children_ages: nil, rooms: nil, currency: nil, pos_country: nil)
|
|
20
|
+
@http.get("/v1/booking/property", {
|
|
21
|
+
property_id: property_id, url: url, check_in: check_in, check_out: check_out,
|
|
22
|
+
adults: adults, children: children, children_ages: children_ages, rooms: rooms,
|
|
23
|
+
currency: currency, pos_country: pos_country,
|
|
24
|
+
})
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def get_reviews(property_id: nil, url: nil, page: nil, language: nil, include_review_details: nil)
|
|
28
|
+
@http.get("/v1/booking/reviews", {
|
|
29
|
+
property_id: property_id, url: url, page: page,
|
|
30
|
+
language: language, include_review_details: include_review_details,
|
|
31
|
+
})
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
module Scrapio
|
|
2
|
+
module Resources
|
|
3
|
+
class FastSearch
|
|
4
|
+
def initialize(http) = @http = http
|
|
5
|
+
|
|
6
|
+
def search(search, country_code: nil, language: nil, page: nil)
|
|
7
|
+
@http.get("/v1/fast-search", { search: search, country_code: country_code, language: language, page: page })
|
|
8
|
+
end
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
module Scrapio
|
|
2
|
+
module Resources
|
|
3
|
+
# POST /v1/map -- site-map/URL-discovery workflow. Untyped request/response,
|
|
4
|
+
# same reasoning as Search.
|
|
5
|
+
class Map
|
|
6
|
+
def initialize(http) = @http = http
|
|
7
|
+
|
|
8
|
+
def execute(request)
|
|
9
|
+
@http.post("/v1/map", request)
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
module Scrapio
|
|
2
|
+
module Resources
|
|
3
|
+
# POST /v1/search -- a multi-provider search workflow (currently resolves
|
|
4
|
+
# through Google, requires a Pro plan or higher). Untyped request/response
|
|
5
|
+
# since the workflow orchestrator's payload is more flexible than a fixed
|
|
6
|
+
# schema.
|
|
7
|
+
class Search
|
|
8
|
+
def initialize(http) = @http = http
|
|
9
|
+
|
|
10
|
+
def execute(request)
|
|
11
|
+
@http.post("/v1/search", request)
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -3,6 +3,11 @@ module Scrapio
|
|
|
3
3
|
class Walmart
|
|
4
4
|
def initialize(http) = @http = http
|
|
5
5
|
|
|
6
|
+
# @deprecated GET /v1/walmart/product is temporarily disabled
|
|
7
|
+
# server-side as of 2026-07-11 (frequent PerimeterX-related blocks on
|
|
8
|
+
# individual Walmart product pages) and will 404 until it's
|
|
9
|
+
# re-enabled. `search` is unaffected. See
|
|
10
|
+
# specs/product/walmart-api.md in the main repo.
|
|
6
11
|
def get_product(product_id, country: nil)
|
|
7
12
|
@http.get("/v1/walmart/product", { product_id: product_id, country: country })
|
|
8
13
|
end
|
metadata
CHANGED
|
@@ -1,18 +1,17 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: scrapio
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Scrapio
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: bin
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date:
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
12
11
|
dependencies: []
|
|
13
12
|
description: Fetch, crawl, search, and extract structured data from any URL. Includes
|
|
14
|
-
Google Search, YouTube
|
|
15
|
-
and async jobs.
|
|
13
|
+
Google and Fast Search, YouTube, Amazon, Walmart, Booking.com, and Agoda data, browser
|
|
14
|
+
automation, and async jobs.
|
|
16
15
|
email: support@scrapio.dev
|
|
17
16
|
executables: []
|
|
18
17
|
extensions: []
|
|
@@ -23,12 +22,17 @@ files:
|
|
|
23
22
|
- lib/scrapio/client.rb
|
|
24
23
|
- lib/scrapio/errors.rb
|
|
25
24
|
- lib/scrapio/http_client.rb
|
|
25
|
+
- lib/scrapio/resources/agoda.rb
|
|
26
26
|
- lib/scrapio/resources/amazon.rb
|
|
27
|
+
- lib/scrapio/resources/booking.rb
|
|
27
28
|
- lib/scrapio/resources/crawl.rb
|
|
29
|
+
- lib/scrapio/resources/fast_search.rb
|
|
28
30
|
- lib/scrapio/resources/fetch.rb
|
|
29
31
|
- lib/scrapio/resources/google.rb
|
|
30
32
|
- lib/scrapio/resources/interact.rb
|
|
31
33
|
- lib/scrapio/resources/jobs.rb
|
|
34
|
+
- lib/scrapio/resources/map.rb
|
|
35
|
+
- lib/scrapio/resources/search.rb
|
|
32
36
|
- lib/scrapio/resources/walmart.rb
|
|
33
37
|
- lib/scrapio/resources/youtube.rb
|
|
34
38
|
homepage: https://scrapio.dev
|
|
@@ -38,7 +42,6 @@ metadata:
|
|
|
38
42
|
homepage_uri: https://scrapio.dev
|
|
39
43
|
source_code_uri: https://github.com/xsronhou/scrapping-tool
|
|
40
44
|
documentation_uri: https://scrapio.dev/docs
|
|
41
|
-
post_install_message:
|
|
42
45
|
rdoc_options: []
|
|
43
46
|
require_paths:
|
|
44
47
|
- lib
|
|
@@ -53,8 +56,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
53
56
|
- !ruby/object:Gem::Version
|
|
54
57
|
version: '0'
|
|
55
58
|
requirements: []
|
|
56
|
-
rubygems_version:
|
|
57
|
-
signing_key:
|
|
59
|
+
rubygems_version: 4.0.8
|
|
58
60
|
specification_version: 4
|
|
59
61
|
summary: Official Ruby SDK for Scrapio
|
|
60
62
|
test_files: []
|