hydrafetch 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +20 -2
- data/lib/hydrafetch/client.rb +15 -3
- data/lib/hydrafetch/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d895f580349c33641dc5f853c8d4c0ee46fa54bf55cf7edcb50093e47997bdf5
|
|
4
|
+
data.tar.gz: 10d8ea933dbd6c8091a616780711e248881c7ba6a83cff3eda0de11d1b122cfd
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b30cee276a93de1f569d53daa0fb039d6fc68d1804b680e31f7f29a76dc7b16ab206527aa10d68eda922c5eb48e526d825d6f6931986d1195bdadece5e46fc70
|
|
7
|
+
data.tar.gz: bac4bf2d4499fc4773bccfe2d59f7441f3f1f38cce49af9bfa971f806638b3d55db97297097a9081b7699f3f3c430f3f839f88a25a88d208ec9bc88f672ae656
|
data/README.md
CHANGED
|
@@ -31,7 +31,7 @@ page = hf.scrape("https://example.com/article")
|
|
|
31
31
|
puts page["markdown"]
|
|
32
32
|
```
|
|
33
33
|
|
|
34
|
-
|
|
34
|
+
Copy your key from [app.hydrafetch.com](https://app.hydrafetch.com). The constructor reads `HYDRAFETCH_API_KEY` when no key is passed.
|
|
35
35
|
|
|
36
36
|
Responses are plain hashes with string keys. Option names are camelCase because they are passed to the API unchanged; client options such as `poll_interval:` and `job_timeout:` are snake_case.
|
|
37
37
|
|
|
@@ -126,10 +126,27 @@ end
|
|
|
126
126
|
hf.brand("stripe.com") # logos, colours, fonts, socials
|
|
127
127
|
hf.logo("stripe.com", theme: "dark", type: "icon") # one asset
|
|
128
128
|
hf.styleguide("stripe.com") # computed design system
|
|
129
|
+
hf.styleguide("https://stripe.com/pricing", maxAge: 0) # one page, read fresh
|
|
129
130
|
```
|
|
130
131
|
|
|
131
132
|
For logos in a browser use [`@hydrafetch/client-sdk`](https://github.com/Hydrafetch/client-sdk) with a publishable key. Those bill against logo pulls rather than credits.
|
|
132
133
|
|
|
134
|
+
## Rating results
|
|
135
|
+
|
|
136
|
+
Tell us when a result was wrong, so it gets looked at. It is free, and rating the same request again replaces the earlier rating.
|
|
137
|
+
|
|
138
|
+
```ruby
|
|
139
|
+
hf.scrape("https://stripe.com/pricing")
|
|
140
|
+
hf.rate_result("bad",
|
|
141
|
+
request_id: hf.last_request_id,
|
|
142
|
+
reasons: ["missing_content"],
|
|
143
|
+
note: "The enterprise tier is missing")
|
|
144
|
+
|
|
145
|
+
hf.rate_result("good", url: "https://stripe.com/pricing") # by URL, last 24 hours
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Reasons are `missing_content`, `wrong_content`, `blocked`, `slow`, `formatting` and `other`.
|
|
149
|
+
|
|
133
150
|
## Error handling
|
|
134
151
|
|
|
135
152
|
All failures raise `Hydrafetch::Error`, carrying the API's error code, HTTP status and request id.
|
|
@@ -181,13 +198,14 @@ hf = Hydrafetch::Client.new("hf_...",
|
|
|
181
198
|
| `extract(urls, **options)` | hash with `"results"` | 5 per URL |
|
|
182
199
|
| `brand(domain)` | brand hash | 5 |
|
|
183
200
|
| `logo(domain, **options)` | logo hash | 1 |
|
|
184
|
-
| `styleguide(
|
|
201
|
+
| `styleguide(domain_or_url, **options)` | design system hash | 10 |
|
|
185
202
|
| `screenshot(url, **options)` | screenshot hash | 5 |
|
|
186
203
|
| `images(url)`, `links(url)` | page assets | 1 |
|
|
187
204
|
| `crawl(url, **options)` | job hash, polled to completion | 1 per page |
|
|
188
205
|
| `batch(urls, **options)` | job hash, polled to completion | 1 per page |
|
|
189
206
|
| `start_crawl`, `start_batch` | job id `String` | 1 per page |
|
|
190
207
|
| `crawl_status(id)`, `batch_status(id)` | job hash | free |
|
|
208
|
+
| `rate_result(rating, request_id:, url:, reasons:, note:)` | rating hash | free |
|
|
191
209
|
|
|
192
210
|
Failed requests are not billed. Pricing does not vary with page difficulty, so there is no render, stealth or proxy option to set.
|
|
193
211
|
|
data/lib/hydrafetch/client.rb
CHANGED
|
@@ -11,7 +11,7 @@ module Hydrafetch
|
|
|
11
11
|
DEFAULT_MAX_RETRIES = 2
|
|
12
12
|
TERMINAL_STATUSES = %w[completed failed cancelled].freeze
|
|
13
13
|
|
|
14
|
-
attr_reader :base_url
|
|
14
|
+
attr_reader :base_url, :last_request_id
|
|
15
15
|
|
|
16
16
|
def initialize(api_key = nil, base_url: DEFAULT_BASE_URL, timeout: DEFAULT_TIMEOUT,
|
|
17
17
|
max_retries: DEFAULT_MAX_RETRIES)
|
|
@@ -32,6 +32,16 @@ module Hydrafetch
|
|
|
32
32
|
post("/v1/web/scrape", { url: url }.merge(compact(options)))
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
+
def rate_result(rating, request_id: nil, url: nil, reasons: nil, note: nil)
|
|
36
|
+
if request_id.nil? && url.nil?
|
|
37
|
+
raise Error.new("Pass request_id: or url: to say which result you are rating.",
|
|
38
|
+
code: "VALIDATION_ERROR", status: 400)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
post("/v1/web/feedback", compact(requestId: request_id, url: url, rating: rating,
|
|
42
|
+
reasons: reasons, note: note))
|
|
43
|
+
end
|
|
44
|
+
|
|
35
45
|
def markdown(url, **options)
|
|
36
46
|
scrape(url, formats: ["markdown"], **options)["markdown"]
|
|
37
47
|
end
|
|
@@ -57,8 +67,9 @@ module Hydrafetch
|
|
|
57
67
|
get("/v1/web/brand/logo", { domain: domain }.merge(compact(options)))
|
|
58
68
|
end
|
|
59
69
|
|
|
60
|
-
def styleguide(
|
|
61
|
-
|
|
70
|
+
def styleguide(domain_or_url, **options)
|
|
71
|
+
key = domain_or_url.match?(%r{\Ahttps?://}i) ? :url : :domain
|
|
72
|
+
get("/v1/web/styleguide", { key => domain_or_url }.merge(compact(options)))
|
|
62
73
|
end
|
|
63
74
|
|
|
64
75
|
def screenshot(url, **options)
|
|
@@ -161,6 +172,7 @@ module Hydrafetch
|
|
|
161
172
|
end
|
|
162
173
|
|
|
163
174
|
res = http_for(uri).request(req)
|
|
175
|
+
@last_request_id = res["x-request-id"] || @last_request_id
|
|
164
176
|
unwrap(res)
|
|
165
177
|
rescue Error => e
|
|
166
178
|
raise unless e.retryable? && attempt < @max_retries
|
data/lib/hydrafetch/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: hydrafetch
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Hydrafetch
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-10-21 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Turn any URL into clean Markdown and structured data. Scrape, map, search,
|
|
14
14
|
extract, brand and bulk crawl through one API.
|