autom 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +161 -0
- data/examples/README.md +11 -0
- data/examples/json.rb +8 -0
- data/examples/request.rb +11 -0
- data/examples/wait_for_job.rb +8 -0
- data/lib/autom/client.rb +266 -0
- data/lib/autom/endpoints.rb +170 -0
- data/lib/autom/errors.rb +56 -0
- data/lib/autom/operations.rb +429 -0
- data/lib/autom/responses.rb +45 -0
- data/lib/autom/serialization.rb +66 -0
- data/lib/autom/transport.rb +27 -0
- data/lib/autom/version.rb +5 -0
- data/lib/autom.rb +14 -0
- data/sig/autom.rbs +101 -0
- metadata +65 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 84b19e6c4244a0ad7d147538852876164759d98c949cc7c71061d92b32693088
|
|
4
|
+
data.tar.gz: f9b75389ba4bf76ff47414faeb97fa000ca729b434a6cc0d6fe64e7e74b09b95
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: c59e4da5bb8076727ca60db59edc5383db46dedc5cc7913c3e0e55d484386e690c3342bf3e172cd6039717f3a289b5cfd90e8c719f6b6f082b5c9a6b8141716c
|
|
7
|
+
data.tar.gz: a0755f8237883ccdca7aea241e88f69d760230f7f1710cae0e1f2c678f930ba4458fafc587a783d9e8e569e31ab825bbf37a412d37f5a9b4b62fb1c6e86cb5b1
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Autom
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# Autom Ruby SDK
|
|
2
|
+
|
|
3
|
+
The official Ruby SDK for the [Autom](https://www.autom.dev) SERP API.
|
|
4
|
+
|
|
5
|
+
Google, Bing and Brave search results as structured JSON — web, maps, news, shopping, images, jobs and AI Mode.
|
|
6
|
+
|
|
7
|
+
The SDK exposes **23 methods**, one per documented endpoint, plus a generic `request`. It only uses the Ruby standard library.
|
|
8
|
+
|
|
9
|
+
## Requirements
|
|
10
|
+
|
|
11
|
+
- Ruby 3.1 or newer
|
|
12
|
+
|
|
13
|
+
## Installation
|
|
14
|
+
|
|
15
|
+
```sh
|
|
16
|
+
gem install autom
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Or in your `Gemfile`:
|
|
20
|
+
|
|
21
|
+
```ruby
|
|
22
|
+
gem "autom"
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Get your API key
|
|
26
|
+
|
|
27
|
+
[Sign up for Autom](https://app.autom.dev/register) or sign in, then copy your API key from the [dashboard](https://app.autom.dev). The client reads `ENV["AUTOM_API_KEY"]` when no key is passed.
|
|
28
|
+
|
|
29
|
+
## Quick start
|
|
30
|
+
|
|
31
|
+
```ruby
|
|
32
|
+
require "autom"
|
|
33
|
+
|
|
34
|
+
client = Autom::Client.new(api_key: ENV.fetch("AUTOM_API_KEY"))
|
|
35
|
+
response = client.google_search(query: "web scraping api")
|
|
36
|
+
puts response.data
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Keyword arguments use the parameter names of the [API reference](https://docs.autom.dev). Unknown names raise `Autom::SerializationError`; use `request` for undocumented parameters.
|
|
40
|
+
|
|
41
|
+
## Async jobs
|
|
42
|
+
|
|
43
|
+
Any call accepts `is_async: true` (useful past the 60-second server limit): the response is then a `job_id`. `wait_for_job` polls `GET /v1/jobs/{id}` until a terminal status (`CompletedJob`, `Error`, `NotFound`, `BadRequest`):
|
|
44
|
+
|
|
45
|
+
```ruby
|
|
46
|
+
submitted = client.google_search(query: "web scraping api", is_async: true)
|
|
47
|
+
job = client.wait_for_job(submitted.data["job_id"]) { |j| puts "#{j['status']} #{j['progress']}%" }
|
|
48
|
+
puts job.data
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## GET or POST
|
|
52
|
+
|
|
53
|
+
Methods use GET when the endpoint supports it (query string). POST sends the same parameters as JSON:
|
|
54
|
+
|
|
55
|
+
```ruby
|
|
56
|
+
client.bing_search(query: "test", request_options: { method: "POST" })
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
`nil` values are omitted, `false` and `0` are sent, and arrays repeat the parameter name.
|
|
60
|
+
|
|
61
|
+
## Any endpoint
|
|
62
|
+
|
|
63
|
+
```ruby
|
|
64
|
+
response = client.request("GET", "/v1/usage")
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Errors
|
|
68
|
+
|
|
69
|
+
```ruby
|
|
70
|
+
begin
|
|
71
|
+
client.google_search(query: "web scraping api")
|
|
72
|
+
rescue Autom::APIError => e
|
|
73
|
+
warn "HTTP #{e.status}: #{e.message}"
|
|
74
|
+
end
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
| Error | When |
|
|
78
|
+
| --- | --- |
|
|
79
|
+
| `Autom::ConfigurationError` | Missing API key, invalid options or unsupported method. |
|
|
80
|
+
| `Autom::SerializationError` | A required parameter is missing, unknown or can't be sent. |
|
|
81
|
+
| `Autom::TransportError` | Network failure or timeout. |
|
|
82
|
+
| `Autom::APIError` | Non-2xx status (`status`, `error_message`, `response_body`, `raw_response`). |
|
|
83
|
+
| `Autom::ResponseDecodingError` | A success response could not be decoded. |
|
|
84
|
+
|
|
85
|
+
All of them inherit from `Autom::Error`.
|
|
86
|
+
|
|
87
|
+
## Retries
|
|
88
|
+
|
|
89
|
+
Requests are retried up to 3 times after a network error, a `408`, `500`, `502`, `503` or `504`, or a rate limit — which the API returns as `401` (or `429`) with a `Rate limit exceeded` message. Per-key quota errors (`…: quota daily`) are not retried, nor are `400`, `402`, `404` or client-side timeouts. The delay starts at 1 second and grows 1.5× per retry (capped at 30 seconds); a `Retry-After` header wins.
|
|
90
|
+
|
|
91
|
+
```ruby
|
|
92
|
+
client = Autom::Client.new(max_retries: 5, retry_delay: 0.5)
|
|
93
|
+
client.google_search(query: "web scraping api", request_options: { max_retries: 0 }) # one request without retries
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Batch requests
|
|
97
|
+
|
|
98
|
+
`batch` runs the block for every item on 10 threads (`concurrency:`) and returns one `BatchResult` per item, in input order. A failing item never stops the others.
|
|
99
|
+
|
|
100
|
+
```ruby
|
|
101
|
+
inputs = ["web scraping api", "serp api", "rank tracker"]
|
|
102
|
+
results = client.batch(inputs, concurrency: 5) { |value| client.google_search(query: value) }
|
|
103
|
+
results.each do |result|
|
|
104
|
+
if result.ok?
|
|
105
|
+
puts result.value.data
|
|
106
|
+
else
|
|
107
|
+
warn "#{inputs[result.index]}: #{result.error.message}"
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
| Constant | Default | Description |
|
|
113
|
+
| --- | --- | --- |
|
|
114
|
+
| `Autom::Client::DEFAULT_CONCURRENCY` | `10` | Tasks run at the same time by batch |
|
|
115
|
+
| `Autom::Client::DEFAULT_TIMEOUT` | `120` | Request timeout (seconds) |
|
|
116
|
+
| `Autom::Client::MAX_RETRIES` | `3` | Retry attempts after a network error, a 408/5xx or a rate limit |
|
|
117
|
+
| `Autom::Client::RETRY_BACKOFF_FACTOR` | `1.5` | Exponential backoff multiplier |
|
|
118
|
+
| `Autom::Client::DEFAULT_RETRY_DELAY` | `1.0` | Delay before the first retry (seconds) |
|
|
119
|
+
|
|
120
|
+
## Timeouts and custom transports
|
|
121
|
+
|
|
122
|
+
```ruby
|
|
123
|
+
client = Autom::Client.new(timeout: 60) # seconds, default 120
|
|
124
|
+
client.google_search(query: "web scraping api", request_options: { timeout: 30 })
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
A transport is any object with `call(method:, url:, headers:, body:, timeout:)` returning `[status, headers, body]`; the default is `Autom::NetHTTPTransport`.
|
|
128
|
+
|
|
129
|
+
## Endpoints
|
|
130
|
+
|
|
131
|
+
23 methods, generated from the [OpenAPI document](https://app.autom.dev/openapi).
|
|
132
|
+
|
|
133
|
+
| Method | HTTP | Description |
|
|
134
|
+
| --- | --- | --- |
|
|
135
|
+
| `bing_search` | GET / POST `/v1/bing/search` | Query Bing Search via API and retrieve structured organic results (title, link, snippet, position). |
|
|
136
|
+
| `brave_images` | GET / POST `/v1/brave/images` | Search Brave Images via API and retrieve structured image results with pagination and SafeSearch. |
|
|
137
|
+
| `brave_search` | GET / POST `/v1/brave/search` | Search Brave Search via API and retrieve structured organic results with pagination. |
|
|
138
|
+
| `brave_videos` | GET / POST `/v1/brave/videos` | Search Brave Videos via API and retrieve structured video results with pagination and SafeSearch. |
|
|
139
|
+
| `duckduckgo_search` | GET / POST `/v1/duckduckgo/search` | Query DuckDuckGo via API and retrieve structured organic results with region and language. |
|
|
140
|
+
| `finder_google_countries` | GET `/v1/finder/google-countries` | Search Google countries and retrieve country codes (free Finder endpoint). |
|
|
141
|
+
| `finder_google_languages` | GET `/v1/finder/google-languages` | Search Google languages and retrieve language codes (free Finder endpoint). |
|
|
142
|
+
| `finder_google_locations` | GET `/v1/finder/google-locations` | Find supported Google locations and retrieve location IDs and country codes (free endpoint). |
|
|
143
|
+
| `google_images` | GET / POST `/v1/google/images` | Search Google Images via API and retrieve image results (URLs, titles, sources, dimensions). |
|
|
144
|
+
| `google_jobs` | GET / POST `/v1/google/jobs` | Scrape Google Jobs results via API and retrieve structured job listings with title, company, location, and pagination. |
|
|
145
|
+
| `google_maps` | GET / POST `/v1/google/maps` | Search Google Maps via API and retrieve structured place results (address, website, phone). |
|
|
146
|
+
| `google_news` | GET / POST `/v1/google/news` | Search Google News via API and retrieve structured news results with pagination. |
|
|
147
|
+
| `google_search` | GET / POST `/v1/google/search` | Scrape Google Search results via API and retrieve structured SERP data with pagination. |
|
|
148
|
+
| `google_search_ai` | GET / POST `/v1/google/search/ai` | Query Google AI Mode via API. JSON with text_blocks, references, and shopping_results. |
|
|
149
|
+
| `google_search_autocomplete` | GET / POST `/v1/google/search/autocomplete` | Get Google Search autocomplete suggestions via API for a given query. |
|
|
150
|
+
| `google_search_light` | GET / POST `/v1/google/search/light` | Scrape lightweight Google Search results via API: fast, cost-efficient, organic results only. |
|
|
151
|
+
| `google_shopping` | GET / POST `/v1/google/shopping` | Scrape Google Shopping results via API and retrieve structured product listings with prices, ratings, and merchant data. |
|
|
152
|
+
| `google_videos` | GET / POST `/v1/google/videos` | Search Google Videos via API and retrieve structured video results with metadata. |
|
|
153
|
+
| `jobs` | GET `/v1/jobs/{id}` | Poll the status, progress (0–100) and result of a job started with is_async=true. Polling is free. Poll until status is terminal: Completed… |
|
|
154
|
+
| `search` | GET / POST `/v1/search` | One call walks engines in order and returns the first organic results. engines=auto tries Google, then Brave, Bing, DuckDuckGo, Yahoo, and… |
|
|
155
|
+
| `stats` | GET `/v1/stats` | Infrastructure health for an engine or endpoint over the last hour. Not scoped to your account or tenant. Enabled per account: HTTP 403 mea… |
|
|
156
|
+
| `usage` | GET `/v1/usage` | Retrieve real-time usage statistics and credit balance for your Autom account. |
|
|
157
|
+
| `yahoo_search` | GET / POST `/v1/yahoo/search` | Query Yahoo via API and retrieve structured organic results with country and language. |
|
|
158
|
+
|
|
159
|
+
## License
|
|
160
|
+
|
|
161
|
+
MIT — see [LICENSE](LICENSE).
|
data/examples/README.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Examples
|
|
2
|
+
|
|
3
|
+
```sh
|
|
4
|
+
gem install autom
|
|
5
|
+
export AUTOM_API_KEY="YOUR_API_KEY"
|
|
6
|
+
ruby examples/json.rb
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
- `json.rb` — call the featured endpoint and print the JSON result.
|
|
10
|
+
- `request.rb` — call any path with `request` and handle API errors.
|
|
11
|
+
- `wait_for_job.rb` — start an async job and wait for its result.
|
data/examples/json.rb
ADDED
data/examples/request.rb
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "autom"
|
|
4
|
+
|
|
5
|
+
client = Autom::Client.new # reads AUTOM_API_KEY
|
|
6
|
+
begin
|
|
7
|
+
response = client.request("GET", "/v1/usage")
|
|
8
|
+
puts response.data if response.json?
|
|
9
|
+
rescue Autom::APIError => e
|
|
10
|
+
warn "HTTP #{e.status}: #{e.message}"
|
|
11
|
+
end
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "autom"
|
|
4
|
+
|
|
5
|
+
client = Autom::Client.new # reads AUTOM_API_KEY
|
|
6
|
+
submitted = client.google_search(query: "web scraping api", is_async: true)
|
|
7
|
+
job = client.wait_for_job(submitted.data["job_id"]) { |j| puts "#{j['status']} #{j['progress']}%" }
|
|
8
|
+
puts job.data
|
data/lib/autom/client.rb
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Autom
|
|
6
|
+
# Official Autom API client. Create one instance and reuse it.
|
|
7
|
+
class Client
|
|
8
|
+
include Operations
|
|
9
|
+
|
|
10
|
+
DEFAULT_BASE_URL = "https://api.autom.dev"
|
|
11
|
+
DEFAULT_TIMEOUT = 120
|
|
12
|
+
# Tasks run at the same time by #batch.
|
|
13
|
+
DEFAULT_CONCURRENCY = 10
|
|
14
|
+
# Retries after a network error or a 408, 429, 500, 502, 503 or 504 response.
|
|
15
|
+
MAX_RETRIES = 3
|
|
16
|
+
# Each retry waits this many times longer than the previous one (a Retry-After header wins).
|
|
17
|
+
RETRY_BACKOFF_FACTOR = 1.5
|
|
18
|
+
# Seconds before the first retry.
|
|
19
|
+
DEFAULT_RETRY_DELAY = 1.0
|
|
20
|
+
MAX_RETRY_DELAY = 30.0
|
|
21
|
+
RETRYABLE_STATUSES = [408, 500, 502, 503, 504].freeze
|
|
22
|
+
API_KEY_ENVIRONMENT_VARIABLE = "AUTOM_API_KEY"
|
|
23
|
+
JOB_TERMINAL_STATUSES = %w[CompletedJob Error NotFound BadRequest].freeze
|
|
24
|
+
ERROR_MESSAGE_KEYS = %w[error message error_message detail].freeze
|
|
25
|
+
|
|
26
|
+
# @param api_key [String, nil] defaults to ENV["AUTOM_API_KEY"]
|
|
27
|
+
# @param base_url [String] API origin
|
|
28
|
+
# @param timeout [Numeric, nil] seconds per request; nil disables it
|
|
29
|
+
# @param headers [Hash] extra headers for every request
|
|
30
|
+
# @param max_retries [Integer] retries after a network error or a 408/429/5xx response; 0 disables them
|
|
31
|
+
# @param retry_delay [Numeric] seconds before the first retry, growing 1.5x per retry
|
|
32
|
+
# @param transport [#call] custom transport, see {NetHTTPTransport}
|
|
33
|
+
def initialize(api_key: nil, base_url: DEFAULT_BASE_URL, timeout: DEFAULT_TIMEOUT, headers: {},
|
|
34
|
+
max_retries: MAX_RETRIES, retry_delay: DEFAULT_RETRY_DELAY, transport: nil)
|
|
35
|
+
api_key = ENV.fetch(API_KEY_ENVIRONMENT_VARIABLE, nil) if api_key.nil?
|
|
36
|
+
if !api_key.is_a?(String) || api_key.strip.empty?
|
|
37
|
+
raise ConfigurationError,
|
|
38
|
+
"A non-blank Autom API key is required. Pass api_key: or set #{API_KEY_ENVIRONMENT_VARIABLE}."
|
|
39
|
+
end
|
|
40
|
+
raise ConfigurationError, "The Autom API key cannot be used in an HTTP header." if api_key.match?(/[\r\n]/)
|
|
41
|
+
|
|
42
|
+
uri = URI.parse(base_url.to_s)
|
|
43
|
+
unless %w[http https].include?(uri.scheme) && uri.host
|
|
44
|
+
raise ConfigurationError, "base_url must be an absolute http or https URL."
|
|
45
|
+
end
|
|
46
|
+
raise ConfigurationError, "timeout must be a positive number or nil." if timeout && !(timeout.is_a?(Numeric) && timeout.positive?)
|
|
47
|
+
raise ConfigurationError, "transport must respond to #call." if transport && !transport.respond_to?(:call)
|
|
48
|
+
|
|
49
|
+
@api_key = api_key
|
|
50
|
+
@base_url = base_url.to_s.sub(%r{/+\z}, "")
|
|
51
|
+
@timeout = timeout
|
|
52
|
+
@headers = headers.to_h { |name, value| [name.to_s, value.to_s] }.freeze
|
|
53
|
+
raise ConfigurationError, "max_retries must be a non-negative Integer." unless max_retries.is_a?(Integer) && max_retries >= 0
|
|
54
|
+
raise ConfigurationError, "retry_delay must be a non-negative number." unless retry_delay.is_a?(Numeric) && retry_delay >= 0
|
|
55
|
+
|
|
56
|
+
@max_retries = max_retries
|
|
57
|
+
@retry_delay = retry_delay
|
|
58
|
+
@transport = transport || NetHTTPTransport.new
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def inspect
|
|
62
|
+
"#<#{self.class.name} base_url=#{@base_url.inspect}>"
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Sends any request, including endpoints released after this SDK version.
|
|
66
|
+
# GET parameters travel in the query string; POST parameters travel in a JSON body.
|
|
67
|
+
# @return [JsonResponse, BinaryResponse]
|
|
68
|
+
def request(method, path, params = nil, timeout: nil, headers: nil, max_retries: nil)
|
|
69
|
+
send_request(method.to_s.upcase, path, params, { timeout: timeout, headers: headers, max_retries: max_retries }, :any)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Runs the block for every item on +concurrency+ threads (default 10) and returns one {BatchResult} per item,
|
|
73
|
+
# in input order. A failing item never stops the others; the client is thread-safe.
|
|
74
|
+
#
|
|
75
|
+
# results = client.batch(ids, concurrency: 5) { |id| client.some_endpoint(query: id) }
|
|
76
|
+
# @return [Array<BatchResult>]
|
|
77
|
+
def batch(items, concurrency: DEFAULT_CONCURRENCY)
|
|
78
|
+
raise ConfigurationError, "concurrency must be a positive Integer." unless concurrency.is_a?(Integer) && concurrency.positive?
|
|
79
|
+
raise ArgumentError, "batch needs a block" unless block_given?
|
|
80
|
+
|
|
81
|
+
list = items.to_a
|
|
82
|
+
results = Array.new(list.size)
|
|
83
|
+
queue = Queue.new
|
|
84
|
+
list.each_with_index { |item, index| queue << [item, index] }
|
|
85
|
+
queue.close
|
|
86
|
+
workers = Array.new([concurrency, list.size].min) do
|
|
87
|
+
Thread.new do
|
|
88
|
+
while (job = queue.pop)
|
|
89
|
+
item, index = job
|
|
90
|
+
results[index] = begin
|
|
91
|
+
BatchResult.new(index: index, value: yield(item), error: nil)
|
|
92
|
+
rescue StandardError => e
|
|
93
|
+
BatchResult.new(index: index, value: nil, error: e)
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
workers.each(&:join)
|
|
99
|
+
results
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Polls GET /v1/jobs/{id} until the job reaches a terminal status. The delay starts at +interval+ seconds
|
|
103
|
+
# and grows 1.5x per poll up to 60 s; +timeout+ defaults to 12 hours.
|
|
104
|
+
# @yieldparam job [Hash] the latest job document after every poll
|
|
105
|
+
# @return [JsonResponse]
|
|
106
|
+
def wait_for_job(job_id, interval: 3, timeout: 43_200)
|
|
107
|
+
raise SerializationError.new("A non-blank job id is required.", "id") if job_id.to_s.strip.empty?
|
|
108
|
+
|
|
109
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
110
|
+
loop do
|
|
111
|
+
last = request("GET", "/v1/jobs/{id}", { "id" => job_id })
|
|
112
|
+
yield last.data if block_given?
|
|
113
|
+
return last if last.data.is_a?(Hash) && JOB_TERMINAL_STATUSES.include?(last.data["status"])
|
|
114
|
+
|
|
115
|
+
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
116
|
+
raise JobTimeoutError.new(job_id, last) if elapsed + interval > timeout
|
|
117
|
+
|
|
118
|
+
sleep(interval)
|
|
119
|
+
interval = (interval * 1.5).clamp(interval, [interval, 60].max)
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
private
|
|
124
|
+
|
|
125
|
+
def call_operation(name, params, options)
|
|
126
|
+
endpoint = ENDPOINTS.fetch(name)
|
|
127
|
+
options = (options || {}).transform_keys(&:to_sym)
|
|
128
|
+
method = (options[:method] || endpoint[:methods].first).to_s.upcase
|
|
129
|
+
unless endpoint[:methods].include?(method)
|
|
130
|
+
raise ConfigurationError, "#{endpoint[:path]} supports #{endpoint[:methods].join(', ')}, not #{method}."
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
params = params.transform_keys(&:to_s)
|
|
134
|
+
unknown = params.keys - endpoint[:params]
|
|
135
|
+
unless unknown.empty?
|
|
136
|
+
raise SerializationError.new(
|
|
137
|
+
"Unknown parameter(s) for #{name}: #{unknown.join(', ')}. Use #request for undocumented parameters.",
|
|
138
|
+
unknown.first
|
|
139
|
+
)
|
|
140
|
+
end
|
|
141
|
+
endpoint[:required].each do |required|
|
|
142
|
+
raise SerializationError.new("The \"#{required}\" parameter is required.", required) if params[required].nil?
|
|
143
|
+
end
|
|
144
|
+
send_request(method, endpoint[:path], params, options, endpoint[:response])
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def send_request(method, path, params, options, expected)
|
|
148
|
+
raise ConfigurationError, "Unsupported HTTP method: #{method.inspect}." unless %w[GET POST].include?(method)
|
|
149
|
+
raise ConfigurationError, 'The request path must start with "/".' unless path.is_a?(String) && path.start_with?("/")
|
|
150
|
+
|
|
151
|
+
path, pairs, body = Serialization.serialize(method, path, params)
|
|
152
|
+
url = @base_url + path
|
|
153
|
+
url += "?#{Serialization.encode_query(pairs)}" unless pairs.empty?
|
|
154
|
+
headers = {
|
|
155
|
+
"Accept" => "application/json, */*;q=0.8",
|
|
156
|
+
"User-Agent" => "autom-ruby/#{VERSION}"
|
|
157
|
+
}.merge(@headers).merge((options[:headers] || {}).to_h { |k, v| [k.to_s, v.to_s] })
|
|
158
|
+
headers["x-api-key"] = @api_key
|
|
159
|
+
payload = nil
|
|
160
|
+
if body
|
|
161
|
+
headers["Content-Type"] = "application/json"
|
|
162
|
+
payload = JSON.generate(body)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
max_retries = options[:max_retries] || @max_retries
|
|
166
|
+
attempt = 0
|
|
167
|
+
loop do
|
|
168
|
+
begin
|
|
169
|
+
status, response_headers, response_body = @transport.call(
|
|
170
|
+
method: method, url: url, headers: headers, body: payload, timeout: options[:timeout] || @timeout
|
|
171
|
+
)
|
|
172
|
+
rescue Timeout::Error => e
|
|
173
|
+
raise TransportError, "Autom request timed out: #{e.message}"
|
|
174
|
+
rescue StandardError => e
|
|
175
|
+
raise TransportError, "Autom request failed before an HTTP response was received: #{e.message}" if attempt >= max_retries
|
|
176
|
+
|
|
177
|
+
sleep(retry_delay(attempt, nil))
|
|
178
|
+
attempt += 1
|
|
179
|
+
next
|
|
180
|
+
end
|
|
181
|
+
raw = build_raw_response(status, response_headers, response_body)
|
|
182
|
+
return decode(raw, expected) if (200..299).cover?(raw.status)
|
|
183
|
+
raise api_error(raw) unless retryable?(raw) && attempt < max_retries
|
|
184
|
+
|
|
185
|
+
sleep(retry_delay(attempt, raw.headers["retry-after"]&.first))
|
|
186
|
+
attempt += 1
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# Server errors and plan rate limits (401/429 "Rate limit exceeded") are retried; per-key quotas are not.
|
|
191
|
+
def retryable?(raw)
|
|
192
|
+
return true if RETRYABLE_STATUSES.include?(raw.status)
|
|
193
|
+
return false unless [401, 429].include?(raw.status)
|
|
194
|
+
|
|
195
|
+
message = api_error(raw).error_message.to_s.downcase
|
|
196
|
+
return false if message.include?("quota")
|
|
197
|
+
|
|
198
|
+
message.include?("rate limit") || (raw.status == 429 && message.empty?)
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def retry_delay(attempt, retry_after)
|
|
202
|
+
seconds = Float(retry_after, exception: false) if retry_after
|
|
203
|
+
return [seconds, 60].min if seconds && seconds >= 0
|
|
204
|
+
|
|
205
|
+
[@retry_delay * (RETRY_BACKOFF_FACTOR**attempt), MAX_RETRY_DELAY].min
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def build_raw_response(status, headers, body)
|
|
209
|
+
normalized = {}
|
|
210
|
+
(headers || {}).each { |name, values| normalized[name.to_s.downcase] = Array(values).map(&:to_s).freeze }
|
|
211
|
+
RawResponse.new(
|
|
212
|
+
status: status.to_i,
|
|
213
|
+
headers: normalized.freeze,
|
|
214
|
+
content_type: normalized["content-type"]&.first,
|
|
215
|
+
body: body.to_s.b
|
|
216
|
+
)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def api_error(raw)
|
|
220
|
+
parsed = parse_json(raw.body)
|
|
221
|
+
message = nil
|
|
222
|
+
if parsed.is_a?(Hash)
|
|
223
|
+
ERROR_MESSAGE_KEYS.each do |key|
|
|
224
|
+
value = parsed[key]
|
|
225
|
+
if value.is_a?(String) && !value.empty?
|
|
226
|
+
message = value
|
|
227
|
+
break
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
APIError.new(raw_response: raw, response_body: parsed == NOT_JSON ? nil : parsed, error_message: message)
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
def decode(raw, expected)
|
|
235
|
+
unless json_media_type?(raw.content_type)
|
|
236
|
+
if expected == :json
|
|
237
|
+
raise ResponseDecodingError.new("Autom returned a successful non-JSON response for a JSON operation.", raw)
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
return BinaryResponse.new(bytes: raw.body, content_type: raw.content_type, raw_response: raw)
|
|
241
|
+
end
|
|
242
|
+
data = parse_json(raw.body)
|
|
243
|
+
if data == NOT_JSON
|
|
244
|
+
raise ResponseDecodingError.new("Autom returned a successful JSON response that could not be decoded.", raw)
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
JsonResponse.new(data: data, raw_response: raw)
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
NOT_JSON = Object.new.freeze
|
|
251
|
+
private_constant :NOT_JSON
|
|
252
|
+
|
|
253
|
+
def parse_json(body)
|
|
254
|
+
JSON.parse(body.dup.force_encoding(Encoding::UTF_8))
|
|
255
|
+
rescue JSON::ParserError, EncodingError
|
|
256
|
+
NOT_JSON
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
def json_media_type?(content_type)
|
|
260
|
+
return false if content_type.nil?
|
|
261
|
+
|
|
262
|
+
media_type = content_type.split(";", 2).first.strip.downcase
|
|
263
|
+
media_type == "application/json" || media_type.end_with?("+json")
|
|
264
|
+
end
|
|
265
|
+
end
|
|
266
|
+
end
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Generated from the Autom OpenAPI document. Do not edit by hand.
|
|
4
|
+
|
|
5
|
+
module Autom
|
|
6
|
+
# Path, methods, response kind and parameter names of every documented operation.
|
|
7
|
+
ENDPOINTS = {
|
|
8
|
+
"bing_search" => {
|
|
9
|
+
path: "/v1/bing/search",
|
|
10
|
+
methods: ["GET", "POST"].freeze,
|
|
11
|
+
response: :json,
|
|
12
|
+
required: ["query"].freeze,
|
|
13
|
+
params: ["query", "page", "num", "location", "lat", "lon", "mkt", "cc", "fields", "is_async", "soft_fail"].freeze
|
|
14
|
+
}.freeze,
|
|
15
|
+
"brave_images" => {
|
|
16
|
+
path: "/v1/brave/images",
|
|
17
|
+
methods: ["GET", "POST"].freeze,
|
|
18
|
+
response: :json,
|
|
19
|
+
required: ["query"].freeze,
|
|
20
|
+
params: ["query", "page", "gl", "hl", "safe", "fields", "is_async", "soft_fail"].freeze
|
|
21
|
+
}.freeze,
|
|
22
|
+
"brave_search" => {
|
|
23
|
+
path: "/v1/brave/search",
|
|
24
|
+
methods: ["GET", "POST"].freeze,
|
|
25
|
+
response: :json,
|
|
26
|
+
required: ["query"].freeze,
|
|
27
|
+
params: ["query", "page", "gl", "hl", "fields", "is_async", "soft_fail"].freeze
|
|
28
|
+
}.freeze,
|
|
29
|
+
"brave_videos" => {
|
|
30
|
+
path: "/v1/brave/videos",
|
|
31
|
+
methods: ["GET", "POST"].freeze,
|
|
32
|
+
response: :json,
|
|
33
|
+
required: ["query"].freeze,
|
|
34
|
+
params: ["query", "page", "gl", "hl", "safe", "fields", "is_async", "soft_fail"].freeze
|
|
35
|
+
}.freeze,
|
|
36
|
+
"duckduckgo_search" => {
|
|
37
|
+
path: "/v1/duckduckgo/search",
|
|
38
|
+
methods: ["GET", "POST"].freeze,
|
|
39
|
+
response: :json,
|
|
40
|
+
required: ["query"].freeze,
|
|
41
|
+
params: ["query", "page", "gl", "hl", "fields", "is_async", "soft_fail"].freeze
|
|
42
|
+
}.freeze,
|
|
43
|
+
"finder_google_countries" => {
|
|
44
|
+
path: "/v1/finder/google-countries",
|
|
45
|
+
methods: ["GET"].freeze,
|
|
46
|
+
response: :json,
|
|
47
|
+
required: ["query"].freeze,
|
|
48
|
+
params: ["query", "fields", "is_async", "soft_fail"].freeze
|
|
49
|
+
}.freeze,
|
|
50
|
+
"finder_google_languages" => {
|
|
51
|
+
path: "/v1/finder/google-languages",
|
|
52
|
+
methods: ["GET"].freeze,
|
|
53
|
+
response: :json,
|
|
54
|
+
required: ["query"].freeze,
|
|
55
|
+
params: ["query", "fields", "is_async", "soft_fail"].freeze
|
|
56
|
+
}.freeze,
|
|
57
|
+
"finder_google_locations" => {
|
|
58
|
+
path: "/v1/finder/google-locations",
|
|
59
|
+
methods: ["GET"].freeze,
|
|
60
|
+
response: :json,
|
|
61
|
+
required: ["query"].freeze,
|
|
62
|
+
params: ["query", "fields", "is_async", "soft_fail"].freeze
|
|
63
|
+
}.freeze,
|
|
64
|
+
"google_images" => {
|
|
65
|
+
path: "/v1/google/images",
|
|
66
|
+
methods: ["GET", "POST"].freeze,
|
|
67
|
+
response: :json,
|
|
68
|
+
required: ["query"].freeze,
|
|
69
|
+
params: ["query", "page", "google_domain", "gl", "hl", "cr", "lr", "async", "safe", "fields", "is_async", "soft_fail"].freeze
|
|
70
|
+
}.freeze,
|
|
71
|
+
"google_jobs" => {
|
|
72
|
+
path: "/v1/google/jobs",
|
|
73
|
+
methods: ["GET", "POST"].freeze,
|
|
74
|
+
response: :json,
|
|
75
|
+
required: ["query"].freeze,
|
|
76
|
+
params: ["query", "uds", "lrad", "ltype", "location", "uule", "google_domain", "gl", "hl", "cr", "lr", "next_page_token", "fields", "is_async", "soft_fail"].freeze
|
|
77
|
+
}.freeze,
|
|
78
|
+
"google_maps" => {
|
|
79
|
+
path: "/v1/google/maps",
|
|
80
|
+
methods: ["GET", "POST"].freeze,
|
|
81
|
+
response: :json,
|
|
82
|
+
required: ["query"].freeze,
|
|
83
|
+
params: ["query", "page", "google_domain", "gl", "hl", "cr", "lr", "ll", "fields", "is_async", "soft_fail"].freeze
|
|
84
|
+
}.freeze,
|
|
85
|
+
"google_news" => {
|
|
86
|
+
path: "/v1/google/news",
|
|
87
|
+
methods: ["GET", "POST"].freeze,
|
|
88
|
+
response: :json,
|
|
89
|
+
required: ["query"].freeze,
|
|
90
|
+
params: ["query", "page", "num", "google_domain", "gl", "hl", "cr", "lr", "fields", "is_async", "soft_fail"].freeze
|
|
91
|
+
}.freeze,
|
|
92
|
+
"google_search" => {
|
|
93
|
+
path: "/v1/google/search",
|
|
94
|
+
methods: ["GET", "POST"].freeze,
|
|
95
|
+
response: :json,
|
|
96
|
+
required: ["query"].freeze,
|
|
97
|
+
params: ["query", "tbs", "safe", "location", "uule", "google_domain", "gl", "hl", "cr", "lr", "page", "num", "fields", "is_async", "soft_fail"].freeze
|
|
98
|
+
}.freeze,
|
|
99
|
+
"google_search_ai" => {
|
|
100
|
+
path: "/v1/google/search/ai",
|
|
101
|
+
methods: ["GET", "POST"].freeze,
|
|
102
|
+
response: :json,
|
|
103
|
+
required: ["query"].freeze,
|
|
104
|
+
params: ["query", "location", "uule", "google_domain", "gl", "hl", "cr", "lr", "fields", "is_async", "soft_fail"].freeze
|
|
105
|
+
}.freeze,
|
|
106
|
+
"google_search_autocomplete" => {
|
|
107
|
+
path: "/v1/google/search/autocomplete",
|
|
108
|
+
methods: ["GET", "POST"].freeze,
|
|
109
|
+
response: :json,
|
|
110
|
+
required: ["query"].freeze,
|
|
111
|
+
params: ["query", "cp", "google_domain", "gl", "hl", "cr", "lr", "fields", "is_async", "soft_fail"].freeze
|
|
112
|
+
}.freeze,
|
|
113
|
+
"google_search_light" => {
|
|
114
|
+
path: "/v1/google/search/light",
|
|
115
|
+
methods: ["GET", "POST"].freeze,
|
|
116
|
+
response: :json,
|
|
117
|
+
required: ["query"].freeze,
|
|
118
|
+
params: ["query", "google_domain", "gl", "hl", "cr", "lr", "fields", "is_async", "soft_fail"].freeze
|
|
119
|
+
}.freeze,
|
|
120
|
+
"google_shopping" => {
|
|
121
|
+
path: "/v1/google/shopping",
|
|
122
|
+
methods: ["GET", "POST"].freeze,
|
|
123
|
+
response: :json,
|
|
124
|
+
required: ["query"].freeze,
|
|
125
|
+
params: ["query", "page", "location", "uule", "google_domain", "gl", "hl", "cr", "lr", "safe", "fields", "is_async", "soft_fail"].freeze
|
|
126
|
+
}.freeze,
|
|
127
|
+
"google_videos" => {
|
|
128
|
+
path: "/v1/google/videos",
|
|
129
|
+
methods: ["GET", "POST"].freeze,
|
|
130
|
+
response: :json,
|
|
131
|
+
required: ["query", "page"].freeze,
|
|
132
|
+
params: ["query", "location", "uule", "google_domain", "gl", "hl", "cr", "lr", "page", "num", "safe", "fields", "is_async", "soft_fail"].freeze
|
|
133
|
+
}.freeze,
|
|
134
|
+
"jobs" => {
|
|
135
|
+
path: "/v1/jobs/{id}",
|
|
136
|
+
methods: ["GET"].freeze,
|
|
137
|
+
response: :json,
|
|
138
|
+
required: ["id"].freeze,
|
|
139
|
+
params: ["id"].freeze
|
|
140
|
+
}.freeze,
|
|
141
|
+
"search" => {
|
|
142
|
+
path: "/v1/search",
|
|
143
|
+
methods: ["GET", "POST"].freeze,
|
|
144
|
+
response: :json,
|
|
145
|
+
required: ["query"].freeze,
|
|
146
|
+
params: ["query", "engines", "gl", "hl", "fields", "is_async", "soft_fail"].freeze
|
|
147
|
+
}.freeze,
|
|
148
|
+
"stats" => {
|
|
149
|
+
path: "/v1/stats",
|
|
150
|
+
methods: ["GET"].freeze,
|
|
151
|
+
response: :json,
|
|
152
|
+
required: [].freeze,
|
|
153
|
+
params: ["engine", "path"].freeze
|
|
154
|
+
}.freeze,
|
|
155
|
+
"usage" => {
|
|
156
|
+
path: "/v1/usage",
|
|
157
|
+
methods: ["GET"].freeze,
|
|
158
|
+
response: :json,
|
|
159
|
+
required: [].freeze,
|
|
160
|
+
params: [].freeze
|
|
161
|
+
}.freeze,
|
|
162
|
+
"yahoo_search" => {
|
|
163
|
+
path: "/v1/yahoo/search",
|
|
164
|
+
methods: ["GET", "POST"].freeze,
|
|
165
|
+
response: :json,
|
|
166
|
+
required: ["query"].freeze,
|
|
167
|
+
params: ["query", "page", "gl", "hl", "fields", "is_async", "soft_fail"].freeze
|
|
168
|
+
}.freeze
|
|
169
|
+
}.freeze
|
|
170
|
+
end
|