minifetch-api 1.5.1 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -18
- package/dist/esm/client.js +213 -44
- package/dist/esm/types/errors.js +44 -0
- package/dist/esm/utils/payment.js +7 -4
- package/dist/esm/utils/validation.js +27 -1
- package/dist/types/client.d.ts +130 -16
- package/dist/types/types/config.d.ts +2 -0
- package/dist/types/types/errors.d.ts +32 -0
- package/dist/types/types/responses.d.ts +35 -0
- package/dist/types/utils/payment.d.ts +2 -2
- package/dist/types/utils/validation.d.ts +9 -0
- package/package.json +4 -2
package/README.md
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
</a>
|
|
6
6
|
</div>
|
|
7
7
|
|
|
8
|
-
**[Minifetch](https://minifetch.com) is a hosted toolkit
|
|
8
|
+
**[Minifetch](https://minifetch.com) is a hosted toolkit for web developers and AI agents. Search, scrape, extract & SEO audit web pages.** Pay per fetch, no subscription.
|
|
9
9
|
|
|
10
10
|
- ✅ **Always pay-per-fetch at competitive prices.**
|
|
11
11
|
- ✅ [Sign up](https://minifetch.com/dashboard) for an account & get free starter credits. 🎉🎉
|
|
@@ -72,17 +72,15 @@ After the Quick Start, you have the following methods to use.
|
|
|
72
72
|
|
|
73
73
|
**Wrap** these methods in a **try/catch** just like in the Quick Start example above. **Code examples** can be also found in the [Github repository /example- directories](https://github.com/Niche-Networks/minifetch-api/).
|
|
74
74
|
|
|
75
|
+
All API methods default to POST unless you set options to `{ method: 'GET' }` and pass in as the second argument.
|
|
76
|
+
|
|
75
77
|
The `checkAndExtract` methods check the target URL's `robots.txt` file to ensure its not blocked and tell us your preferred crawl delay (defaults to 1 second between requests to your domain). So fetching 10 URLs takes at least 10 seconds to complete by default. This is by design, so Minifetch never hammers your server or slows it down for your real users. If you own the site and want to allow Minifetch access or to set custom rules for it, read [How To Unblock Minifetch](https://minifetch.com/tutorials/unblock-minifetch).
|
|
76
78
|
|
|
77
79
|
```js
|
|
78
|
-
await client.
|
|
79
|
-
// Price: $0.
|
|
80
|
-
//
|
|
81
|
-
//
|
|
82
|
-
// WARN/ FAIL result with no black-box scoring. Just deterministic,
|
|
83
|
-
// composable signal you can act on or pipe into an agent.
|
|
84
|
-
// Audit rules are documented in the skill file:
|
|
85
|
-
// https://minifetch.com/skills/seo-page-audit/SKILL.md
|
|
80
|
+
await client.searchByKeyword("green tea");
|
|
81
|
+
// Price: $0.002
|
|
82
|
+
// Keyword web search by Ceramic.ai that returns ranked results as
|
|
83
|
+
// structured JSON: a title, URL, and text snippet per result.
|
|
86
84
|
|
|
87
85
|
await client.checkAndExtractUrlMetadata(url, options);
|
|
88
86
|
// Price: $0.002
|
|
@@ -106,36 +104,45 @@ await client.checkAndExtractUrlLinks(url);
|
|
|
106
104
|
// anchor text variants used for each) and top external domains by
|
|
107
105
|
// link count.
|
|
108
106
|
|
|
109
|
-
await client.checkAndExtractUrlPreview(url);
|
|
110
|
-
// Price: $0.002
|
|
111
|
-
// Extracts all fields used for a page's share previews: the lightweight
|
|
112
|
-
// cards that represent the page on social platforms, chat apps, and AI.
|
|
113
|
-
|
|
114
107
|
await client.checkAndExtractUrlContent(url, options);
|
|
115
108
|
// Price: $0.002
|
|
116
109
|
// For site owners auditing AI readability: returns the clean markdown
|
|
117
110
|
// an LLM extracts from your page after nav, ads, & scripts are stripped.
|
|
118
111
|
// See what survives for AEO and AI search; respects robots.txt.
|
|
119
112
|
// Options: { includeMediaUrls: true } - defaults to false.
|
|
113
|
+
|
|
114
|
+
await client.checkAndExtractUrlPreview(url);
|
|
115
|
+
// Price: $0.002
|
|
116
|
+
// Extracts all fields used for a page's share previews: the lightweight
|
|
117
|
+
// cards that represent the page on social platforms, chat apps, and AI.
|
|
118
|
+
|
|
119
|
+
await client.checkAndRunSeoPageAudit(url);
|
|
120
|
+
// Price: $0.01
|
|
121
|
+
// Runs a full technical SEO audit on your URL. Combines data from
|
|
122
|
+
// the other API endpoints and runs checks that each return a PASS/
|
|
123
|
+
// WARN/ FAIL result with no black-box scoring. Just deterministic,
|
|
124
|
+
// composable signal you can act on or pipe into an agent.
|
|
125
|
+
// Audit rules are documented in the skill file:
|
|
126
|
+
// https://minifetch.com/skills/seo-page-audit/SKILL.md
|
|
120
127
|
```
|
|
121
128
|
|
|
122
129
|
For max control, you can also use the following methods directly:
|
|
123
130
|
```js
|
|
124
131
|
await client.preflightCheck(url, options);
|
|
125
|
-
// Free: check if `minfetch` user agent can access target via robots.txt
|
|
132
|
+
// Free: check if `minfetch` user agent can access target URL via robots.txt
|
|
126
133
|
// Options: { "fresh": true } - bypass 24hr robots.txt cache, defaults to false
|
|
127
134
|
|
|
128
135
|
// Paid methods:
|
|
129
|
-
await client.runSeoPageAudit(url);
|
|
130
136
|
await client.extractUrlMetadata(url, options); // same options as above
|
|
131
137
|
await client.extractUrlLinks(url);
|
|
132
|
-
await client.extractUrlPreview(url);
|
|
133
138
|
await client.extractUrlContent(url, options); // same options as above
|
|
139
|
+
await client.extractUrlPreview(url);
|
|
140
|
+
await client.runSeoPageAudit(url);
|
|
134
141
|
```
|
|
135
142
|
---
|
|
136
143
|
|
|
137
144
|
### Error Types
|
|
138
|
-
When you wrap the functions above in a try/catch, here are some of the errors you may encounter. You are never charged for URLs that are blocked or error.
|
|
145
|
+
When you wrap the functions above in a try/catch, here are some of the errors you may encounter. You are never charged for target URLs that are blocked or error.
|
|
139
146
|
|
|
140
147
|
- **"InvalidURLError: Invalid url ${url}"**
|
|
141
148
|
- The URL is malformed in some way, correct it and try again.
|
package/dist/esm/client.js
CHANGED
|
@@ -1,12 +1,18 @@
|
|
|
1
1
|
import { initConfig } from "./init.js";
|
|
2
|
-
import { validateAndNormalizeUrl } from "./utils/validation.js";
|
|
2
|
+
import { validateAndNormalizeUrl, validateAndNormalizeQuery } from "./utils/validation.js";
|
|
3
3
|
import { handlePayment, handleApiKeyRequest } from "./utils/payment.js";
|
|
4
|
-
import { InvalidUrlError, RobotsBlockedError, PaymentFailedError, ExtractionFailedError, NetworkError, } from "./types/errors.js";
|
|
4
|
+
import { InvalidUrlError, InvalidQueryError, RobotsBlockedError, PaymentFailedError, ExtractionFailedError, SearchFailedError, NetworkError, ConfigurationError, } from "./types/errors.js";
|
|
5
|
+
/** Every endpoint accepts GET (query string) or POST (JSON body). POST is the default. */
|
|
6
|
+
const DEFAULT_METHOD = "POST";
|
|
5
7
|
/**
|
|
6
8
|
* Main Minifetch API client.
|
|
7
9
|
* Supports two auth modes:
|
|
8
10
|
* - x402: crypto micropayments via Coinbase x402 (pass network + privateKey)
|
|
9
11
|
* - apiKey: Stripe-backed credits (pass apiKey: "mf_prod_..." or "mf_dev_...")
|
|
12
|
+
*
|
|
13
|
+
* Every request method accepts an optional `method: "GET" | "POST"` in its
|
|
14
|
+
* options; it defaults to POST (params sent as a JSON body). Pass `method: "GET"`
|
|
15
|
+
* to send params in the query string instead.
|
|
10
16
|
*/
|
|
11
17
|
export class MinifetchClient {
|
|
12
18
|
config;
|
|
@@ -18,23 +24,57 @@ export class MinifetchClient {
|
|
|
18
24
|
this.config = initConfig(config);
|
|
19
25
|
this.baseUrl = this.config.apiBaseUrl;
|
|
20
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* Search the web by keyword (paid endpoint).
|
|
29
|
+
*
|
|
30
|
+
* Unlike the URL-based methods, this takes a search query rather than a URL and
|
|
31
|
+
* proxies to Minifetch's keyword search. Returns ranked results, each with a
|
|
32
|
+
* title, URL, and text snippet. `limit` and `descriptionLength` are convenience
|
|
33
|
+
* knobs the server clamps into range; the effective values (after clamping) are
|
|
34
|
+
* echoed back on `queryParameters`.
|
|
35
|
+
*
|
|
36
|
+
* @param query - keyword(s) to search for (1-50 characters)
|
|
37
|
+
* @param options
|
|
38
|
+
* @param options.limit - number of results, 1-10 (default 10). Out-of-range values are clamped.
|
|
39
|
+
* @param options.descriptionLength - max characters per snippet, 0-5000 (default 750, 0 = titles/URLs only). Out-of-range values are clamped.
|
|
40
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
41
|
+
* @throws {InvalidQueryError} if the query is empty or exceeds 50 characters
|
|
42
|
+
* @throws {SearchFailedError} if the search request fails
|
|
43
|
+
* @throws {PaymentFailedError} if x402 payment fails
|
|
44
|
+
* @throws {NetworkError} various reasons, check README
|
|
45
|
+
*/
|
|
46
|
+
async searchByKeyword(query, options) {
|
|
47
|
+
try {
|
|
48
|
+
const cleanQuery = validateAndNormalizeQuery(query);
|
|
49
|
+
const params = { query: cleanQuery };
|
|
50
|
+
if (options?.limit !== undefined)
|
|
51
|
+
params.limit = options.limit;
|
|
52
|
+
if (options?.descriptionLength !== undefined)
|
|
53
|
+
params.descriptionLength = options.descriptionLength;
|
|
54
|
+
return await this._makeSearchRequest(params, options?.method);
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
return this._rethrowSearchError(error, query, "Keyword search");
|
|
58
|
+
}
|
|
59
|
+
}
|
|
21
60
|
/**
|
|
22
61
|
* Check if URL is allowed by robots.txt (free preflight check — no auth required)
|
|
23
62
|
*
|
|
24
63
|
* @param url
|
|
25
64
|
* @param options
|
|
26
65
|
* @param options.fresh
|
|
66
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
27
67
|
* @throws {InvalidUrlError} if URL is invalid
|
|
28
68
|
* @throws {NetworkError} if request fails
|
|
29
69
|
*/
|
|
30
70
|
async preflightUrlCheck(url, options) {
|
|
31
71
|
try {
|
|
32
72
|
const normalizedUrl = validateAndNormalizeUrl(url);
|
|
33
|
-
const params =
|
|
73
|
+
const params = { url: normalizedUrl };
|
|
34
74
|
if (options?.fresh)
|
|
35
|
-
params.
|
|
36
|
-
const requestUrl =
|
|
37
|
-
const response = await fetch(requestUrl);
|
|
75
|
+
params.fresh = true;
|
|
76
|
+
const { url: requestUrl, init } = this._buildRequest("/api/v1/free/preflight/url-check", params, options?.method ?? DEFAULT_METHOD);
|
|
77
|
+
const response = await fetch(requestUrl, init);
|
|
38
78
|
if (!response.ok) {
|
|
39
79
|
throw new NetworkError(`Preflight check failed: ${response.status} ${response.statusText}`);
|
|
40
80
|
}
|
|
@@ -48,23 +88,34 @@ export class MinifetchClient {
|
|
|
48
88
|
}
|
|
49
89
|
}
|
|
50
90
|
/**
|
|
51
|
-
*
|
|
91
|
+
* INTERNAL / UNDOCUMENTED — deliberately omitted from the README and not part
|
|
92
|
+
* of the public API. The public {@link preflightUrlCheck} hits the FREE
|
|
93
|
+
* endpoint (no payment); this hits the PAID x402 twin at
|
|
94
|
+
* `/api/v1/x402/preflight/url-check` so our own suite can generate paid
|
|
95
|
+
* traffic against it — the x402 Bazaar weights usage for ranking and this
|
|
96
|
+
* refreshes the listing. x402 auth only; there is no session/api-key route
|
|
97
|
+
* for a paid url-check.
|
|
52
98
|
*
|
|
53
99
|
* @param url
|
|
54
|
-
* @
|
|
55
|
-
* @
|
|
56
|
-
* @
|
|
57
|
-
* @throws {
|
|
100
|
+
* @param options
|
|
101
|
+
* @param options.fresh - bypass the 24h robots.txt cache
|
|
102
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
103
|
+
* @throws {ConfigurationError} if the client is not in x402 mode
|
|
104
|
+
* @internal
|
|
58
105
|
*/
|
|
59
|
-
async
|
|
106
|
+
async _exercisePaidUrlCheck(url, options) {
|
|
107
|
+
if (this.config.authMode !== "x402") {
|
|
108
|
+
throw new ConfigurationError("_exercisePaidUrlCheck requires x402 auth (network + privateKey)");
|
|
109
|
+
}
|
|
60
110
|
try {
|
|
61
111
|
const normalizedUrl = validateAndNormalizeUrl(url);
|
|
62
|
-
const params =
|
|
63
|
-
|
|
64
|
-
|
|
112
|
+
const params = { url: normalizedUrl };
|
|
113
|
+
if (options?.fresh)
|
|
114
|
+
params.fresh = true;
|
|
115
|
+
return await this._makeRequest("/preflight/url-check", normalizedUrl, "Paid URL check", params, options?.method);
|
|
65
116
|
}
|
|
66
117
|
catch (error) {
|
|
67
|
-
return this._rethrowError(error, url, "
|
|
118
|
+
return this._rethrowError(error, url, "Paid URL check");
|
|
68
119
|
}
|
|
69
120
|
}
|
|
70
121
|
/**
|
|
@@ -75,6 +126,7 @@ export class MinifetchClient {
|
|
|
75
126
|
* @param options.fields
|
|
76
127
|
* @param options.omitEmpty
|
|
77
128
|
* @param options.includeResponseBody
|
|
129
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
78
130
|
* @throws {InvalidUrlError} if URL is invalid
|
|
79
131
|
* @throws {ExtractionFailedError} various reasons, check README
|
|
80
132
|
* @throws {PaymentFailedError} if x402 payment fails
|
|
@@ -83,15 +135,14 @@ export class MinifetchClient {
|
|
|
83
135
|
async extractUrlMetadata(url, options) {
|
|
84
136
|
try {
|
|
85
137
|
const normalizedUrl = validateAndNormalizeUrl(url);
|
|
86
|
-
const params =
|
|
138
|
+
const params = { url: normalizedUrl };
|
|
87
139
|
if (options?.fields?.length)
|
|
88
|
-
params.
|
|
140
|
+
params.fields = options.fields.join(",");
|
|
89
141
|
if (options?.omitEmpty)
|
|
90
|
-
params.
|
|
142
|
+
params.omitEmpty = true;
|
|
91
143
|
if (options?.includeResponseBody)
|
|
92
|
-
params.
|
|
93
|
-
|
|
94
|
-
return await this._makeRequest(requestUrl, normalizedUrl, "Metadata extraction");
|
|
144
|
+
params.includeResponseBody = true;
|
|
145
|
+
return await this._makeRequest("/extract/url-metadata", normalizedUrl, "Metadata extraction", params, options?.method);
|
|
95
146
|
}
|
|
96
147
|
catch (error) {
|
|
97
148
|
return this._rethrowError(error, url, "Metadata extraction");
|
|
@@ -101,16 +152,18 @@ export class MinifetchClient {
|
|
|
101
152
|
* Extract URL links (paid endpoint)
|
|
102
153
|
*
|
|
103
154
|
* @param url
|
|
155
|
+
* @param options
|
|
156
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
104
157
|
* @throws {InvalidUrlError} if URL is invalid
|
|
105
158
|
* @throws {ExtractionFailedError} various reasons, check README
|
|
106
159
|
* @throws {PaymentFailedError} if x402 payment fails
|
|
107
160
|
* @throws {NetworkError} various reasons, check README
|
|
108
161
|
*/
|
|
109
|
-
async extractUrlLinks(url) {
|
|
162
|
+
async extractUrlLinks(url, options) {
|
|
110
163
|
try {
|
|
111
164
|
const normalizedUrl = validateAndNormalizeUrl(url);
|
|
112
|
-
const
|
|
113
|
-
return await this._makeRequest(
|
|
165
|
+
const params = { url: normalizedUrl };
|
|
166
|
+
return await this._makeRequest("/extract/url-links", normalizedUrl, "Links extraction", params, options?.method);
|
|
114
167
|
}
|
|
115
168
|
catch (error) {
|
|
116
169
|
return this._rethrowError(error, url, "Links extraction");
|
|
@@ -120,16 +173,18 @@ export class MinifetchClient {
|
|
|
120
173
|
* Extract URL preview (paid endpoint)
|
|
121
174
|
*
|
|
122
175
|
* @param url
|
|
176
|
+
* @param options
|
|
177
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
123
178
|
* @throws {InvalidUrlError} if URL is invalid
|
|
124
179
|
* @throws {ExtractionFailedError} various reasons, check README
|
|
125
180
|
* @throws {PaymentFailedError} if x402 payment fails
|
|
126
181
|
* @throws {NetworkError} various reasons, check README
|
|
127
182
|
*/
|
|
128
|
-
async extractUrlPreview(url) {
|
|
183
|
+
async extractUrlPreview(url, options) {
|
|
129
184
|
try {
|
|
130
185
|
const normalizedUrl = validateAndNormalizeUrl(url);
|
|
131
|
-
const
|
|
132
|
-
return await this._makeRequest(
|
|
186
|
+
const params = { url: normalizedUrl };
|
|
187
|
+
return await this._makeRequest("/extract/url-preview", normalizedUrl, "Preview extraction", params, options?.method);
|
|
133
188
|
}
|
|
134
189
|
catch (error) {
|
|
135
190
|
return this._rethrowError(error, url, "Preview extraction");
|
|
@@ -141,6 +196,7 @@ export class MinifetchClient {
|
|
|
141
196
|
* @param url
|
|
142
197
|
* @param options
|
|
143
198
|
* @param options.includeMediaUrls
|
|
199
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
144
200
|
* @throws {InvalidUrlError} if URL is invalid
|
|
145
201
|
* @throws {ExtractionFailedError} various reasons, check README
|
|
146
202
|
* @throws {PaymentFailedError} if x402 payment fails
|
|
@@ -149,25 +205,47 @@ export class MinifetchClient {
|
|
|
149
205
|
async extractUrlContent(url, options) {
|
|
150
206
|
try {
|
|
151
207
|
const normalizedUrl = validateAndNormalizeUrl(url);
|
|
152
|
-
const params =
|
|
208
|
+
const params = { url: normalizedUrl };
|
|
153
209
|
if (options?.includeMediaUrls)
|
|
154
|
-
params.
|
|
155
|
-
|
|
156
|
-
return await this._makeRequest(requestUrl, normalizedUrl, "Content extraction");
|
|
210
|
+
params.includeMediaUrls = true;
|
|
211
|
+
return await this._makeRequest("/extract/url-content", normalizedUrl, "Content extraction", params, options?.method);
|
|
157
212
|
}
|
|
158
213
|
catch (error) {
|
|
159
214
|
return this._rethrowError(error, url, "Content extraction");
|
|
160
215
|
}
|
|
161
216
|
}
|
|
217
|
+
/**
|
|
218
|
+
* Run SEO page audit (paid endpoint)
|
|
219
|
+
*
|
|
220
|
+
* @param url
|
|
221
|
+
* @param options
|
|
222
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
223
|
+
* @throws {InvalidUrlError} if URL is invalid
|
|
224
|
+
* @throws {ExtractionFailedError} various reasons, check README
|
|
225
|
+
* @throws {PaymentFailedError} if x402 payment fails
|
|
226
|
+
* @throws {NetworkError} various reasons, check README
|
|
227
|
+
*/
|
|
228
|
+
async runSeoPageAudit(url, options) {
|
|
229
|
+
try {
|
|
230
|
+
const normalizedUrl = validateAndNormalizeUrl(url);
|
|
231
|
+
const params = { url: normalizedUrl };
|
|
232
|
+
return await this._makeRequest("/run/seo-page-audit", normalizedUrl, "Run SEO page audit", params, options?.method);
|
|
233
|
+
}
|
|
234
|
+
catch (error) {
|
|
235
|
+
return this._rethrowError(error, url, "Run SEO page audit");
|
|
236
|
+
}
|
|
237
|
+
}
|
|
162
238
|
/**
|
|
163
239
|
* Check URL then run SEO page audit in one call.
|
|
164
240
|
* Throws RobotsBlockedError if robots.txt blocks the URL.
|
|
165
241
|
*
|
|
166
242
|
* @param url
|
|
243
|
+
* @param options
|
|
244
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
167
245
|
*/
|
|
168
|
-
async checkAndRunSeoPageAudit(url) {
|
|
246
|
+
async checkAndRunSeoPageAudit(url, options) {
|
|
169
247
|
await this._preflightOrThrow(url);
|
|
170
|
-
return this.runSeoPageAudit(url);
|
|
248
|
+
return this.runSeoPageAudit(url, options);
|
|
171
249
|
}
|
|
172
250
|
/**
|
|
173
251
|
* Check URL then extract metadata in one call.
|
|
@@ -178,6 +256,7 @@ export class MinifetchClient {
|
|
|
178
256
|
* @param options.fields
|
|
179
257
|
* @param options.omitEmpty
|
|
180
258
|
* @param options.includeResponseBody
|
|
259
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
181
260
|
*/
|
|
182
261
|
async checkAndExtractUrlMetadata(url, options) {
|
|
183
262
|
await this._preflightOrThrow(url);
|
|
@@ -188,20 +267,24 @@ export class MinifetchClient {
|
|
|
188
267
|
* Throws RobotsBlockedError if robots.txt blocks the URL.
|
|
189
268
|
*
|
|
190
269
|
* @param url
|
|
270
|
+
* @param options
|
|
271
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
191
272
|
*/
|
|
192
|
-
async checkAndExtractUrlLinks(url) {
|
|
273
|
+
async checkAndExtractUrlLinks(url, options) {
|
|
193
274
|
await this._preflightOrThrow(url);
|
|
194
|
-
return this.extractUrlLinks(url);
|
|
275
|
+
return this.extractUrlLinks(url, options);
|
|
195
276
|
}
|
|
196
277
|
/**
|
|
197
278
|
* Check URL then extract preview in one call.
|
|
198
279
|
* Throws RobotsBlockedError if robots.txt blocks the URL.
|
|
199
280
|
*
|
|
200
281
|
* @param url
|
|
282
|
+
* @param options
|
|
283
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
201
284
|
*/
|
|
202
|
-
async checkAndExtractUrlPreview(url) {
|
|
285
|
+
async checkAndExtractUrlPreview(url, options) {
|
|
203
286
|
await this._preflightOrThrow(url);
|
|
204
|
-
return this.extractUrlPreview(url);
|
|
287
|
+
return this.extractUrlPreview(url, options);
|
|
205
288
|
}
|
|
206
289
|
/**
|
|
207
290
|
* Check URL then extract content in one call.
|
|
@@ -210,6 +293,7 @@ export class MinifetchClient {
|
|
|
210
293
|
* @param url
|
|
211
294
|
* @param options
|
|
212
295
|
* @param options.includeMediaUrls
|
|
296
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
213
297
|
*/
|
|
214
298
|
async checkAndExtractUrlContent(url, options) {
|
|
215
299
|
await this._preflightOrThrow(url);
|
|
@@ -229,17 +313,46 @@ export class MinifetchClient {
|
|
|
229
313
|
return this.config.authMode === "x402" ? `/api/v1/x402${endpoint}` : `/api/v1${endpoint}`;
|
|
230
314
|
}
|
|
231
315
|
/**
|
|
232
|
-
*
|
|
233
|
-
*
|
|
316
|
+
* Encode a request for the wire. GET → params in the query string, no body.
|
|
317
|
+
* POST → params as a JSON body with a Content-Type header. Auth headers
|
|
318
|
+
* (Bearer / x402 payment) are added downstream, not here.
|
|
319
|
+
*
|
|
320
|
+
* @param path - absolute API path (already includes /api/v1[/x402])
|
|
321
|
+
* @param params - request params (string or boolean values)
|
|
322
|
+
* @param method - "GET" or "POST"
|
|
323
|
+
* @returns the full request URL and the fetch init (method + optional body/headers)
|
|
324
|
+
*/
|
|
325
|
+
_buildRequest(path, params, method) {
|
|
326
|
+
if (method === "GET") {
|
|
327
|
+
const qs = new URLSearchParams();
|
|
328
|
+
for (const [key, value] of Object.entries(params))
|
|
329
|
+
qs.set(key, String(value));
|
|
330
|
+
return { url: `${this.baseUrl}${path}?${qs.toString()}`, init: { method: "GET" } };
|
|
331
|
+
}
|
|
332
|
+
return {
|
|
333
|
+
url: `${this.baseUrl}${path}`,
|
|
334
|
+
init: {
|
|
335
|
+
method: "POST",
|
|
336
|
+
headers: { "Content-Type": "application/json" },
|
|
337
|
+
body: JSON.stringify(params),
|
|
338
|
+
},
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* Build the request, dispatch to the correct auth handler, then normalize the
|
|
343
|
+
* response into PaidEndpointResponse.
|
|
234
344
|
* Note: payment field is only present for x402 responses.
|
|
235
345
|
*
|
|
236
|
-
* @param
|
|
346
|
+
* @param endpoint - endpoint path segment, e.g. "/extract/url-metadata"
|
|
237
347
|
* @param normalizedUrl
|
|
238
348
|
* @param label - used in error messages
|
|
349
|
+
* @param params - request params (string or boolean values)
|
|
350
|
+
* @param method - "GET" or "POST" (default POST)
|
|
239
351
|
*/
|
|
240
|
-
async _makeRequest(
|
|
352
|
+
async _makeRequest(endpoint, normalizedUrl, label, params, method = DEFAULT_METHOD) {
|
|
353
|
+
const { url, init } = this._buildRequest(this._paidPath(endpoint), params, method);
|
|
241
354
|
if (this.config.authMode === "x402") {
|
|
242
|
-
const { response, payment } = await handlePayment(
|
|
355
|
+
const { response, payment } = await handlePayment(url, this.config, init);
|
|
243
356
|
if (!response.ok) {
|
|
244
357
|
throw new ExtractionFailedError(normalizedUrl, `${label} failed: ${response.status} ${response.statusText}`);
|
|
245
358
|
}
|
|
@@ -247,7 +360,7 @@ export class MinifetchClient {
|
|
|
247
360
|
return { success: data.success, results: data.results, payment };
|
|
248
361
|
}
|
|
249
362
|
else {
|
|
250
|
-
const { response } = await handleApiKeyRequest(
|
|
363
|
+
const { response } = await handleApiKeyRequest(url, this.config, init);
|
|
251
364
|
if (!response.ok) {
|
|
252
365
|
throw new ExtractionFailedError(normalizedUrl, `${label} failed: ${response.status} ${response.statusText}`);
|
|
253
366
|
}
|
|
@@ -256,6 +369,45 @@ export class MinifetchClient {
|
|
|
256
369
|
return { success: data.success, results: data.results };
|
|
257
370
|
}
|
|
258
371
|
}
|
|
372
|
+
/**
|
|
373
|
+
* Search-specific request path. Mirrors {@link _makeRequest} but for the
|
|
374
|
+
* keyword search endpoint: there is no URL, non-OK responses surface as
|
|
375
|
+
* SearchFailedError (carrying the query), and the server's clamped
|
|
376
|
+
* `queryParameters` echo is preserved rather than dropped.
|
|
377
|
+
*
|
|
378
|
+
* @param params - request params (query + optional limit/descriptionLength)
|
|
379
|
+
* @param method - "GET" or "POST" (default POST)
|
|
380
|
+
*/
|
|
381
|
+
async _makeSearchRequest(params, method = DEFAULT_METHOD) {
|
|
382
|
+
const query = String(params.query);
|
|
383
|
+
const { url, init } = this._buildRequest(this._paidPath("/search/keyword"), params, method);
|
|
384
|
+
if (this.config.authMode === "x402") {
|
|
385
|
+
const { response, payment } = await handlePayment(url, this.config, init);
|
|
386
|
+
if (!response.ok) {
|
|
387
|
+
throw new SearchFailedError(query, `Keyword search failed: ${response.status} ${response.statusText}`, response.status);
|
|
388
|
+
}
|
|
389
|
+
const data = (await response.json());
|
|
390
|
+
return {
|
|
391
|
+
success: data.success,
|
|
392
|
+
queryParameters: data.queryParameters,
|
|
393
|
+
results: data.results,
|
|
394
|
+
payment,
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
else {
|
|
398
|
+
const { response } = await handleApiKeyRequest(url, this.config, init);
|
|
399
|
+
if (!response.ok) {
|
|
400
|
+
throw new SearchFailedError(query, `Keyword search failed: ${response.status} ${response.statusText}`, response.status);
|
|
401
|
+
}
|
|
402
|
+
const data = (await response.json());
|
|
403
|
+
// payment field intentionally omitted for apiKey auth — not applicable
|
|
404
|
+
return {
|
|
405
|
+
success: data.success,
|
|
406
|
+
queryParameters: data.queryParameters,
|
|
407
|
+
results: data.results,
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
}
|
|
259
411
|
/**
|
|
260
412
|
* Preflight check helper — throws RobotsBlockedError if not allowed
|
|
261
413
|
*
|
|
@@ -284,4 +436,21 @@ export class MinifetchClient {
|
|
|
284
436
|
}
|
|
285
437
|
throw new ExtractionFailedError(url, `${label} failed: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
286
438
|
}
|
|
439
|
+
/**
|
|
440
|
+
* Search sibling of {@link _rethrowError}: re-throw known search error types,
|
|
441
|
+
* wrapping anything else in SearchFailedError (which carries the query, not a URL).
|
|
442
|
+
*
|
|
443
|
+
* @param error
|
|
444
|
+
* @param query
|
|
445
|
+
* @param label
|
|
446
|
+
*/
|
|
447
|
+
_rethrowSearchError(error, query, label) {
|
|
448
|
+
if (error instanceof InvalidQueryError ||
|
|
449
|
+
error instanceof SearchFailedError ||
|
|
450
|
+
error instanceof PaymentFailedError ||
|
|
451
|
+
error instanceof NetworkError) {
|
|
452
|
+
throw error;
|
|
453
|
+
}
|
|
454
|
+
throw new SearchFailedError(query, `${label} failed: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
455
|
+
}
|
|
287
456
|
}
|
package/dist/esm/types/errors.js
CHANGED
|
@@ -120,3 +120,47 @@ export class NetworkError extends MinifetchError {
|
|
|
120
120
|
Object.setPrototypeOf(this, NetworkError.prototype);
|
|
121
121
|
}
|
|
122
122
|
}
|
|
123
|
+
/**
|
|
124
|
+
* Thrown when keyword-search query validation fails (empty, or over 50 chars).
|
|
125
|
+
* The query sibling of {@link InvalidUrlError} — searchByKeyword takes a query,
|
|
126
|
+
* not a URL, so bad input surfaces here instead.
|
|
127
|
+
*/
|
|
128
|
+
export class InvalidQueryError extends MinifetchError {
|
|
129
|
+
query;
|
|
130
|
+
/**
|
|
131
|
+
*
|
|
132
|
+
* @param query
|
|
133
|
+
* @param message
|
|
134
|
+
*/
|
|
135
|
+
constructor(query, message) {
|
|
136
|
+
super(message || `Invalid query: ${query}`);
|
|
137
|
+
this.name = "InvalidQueryError";
|
|
138
|
+
this.query = query;
|
|
139
|
+
Object.setPrototypeOf(this, InvalidQueryError.prototype);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Thrown when a keyword search fails (non-OK response or unexpected error).
|
|
144
|
+
* The search sibling of {@link ExtractionFailedError}: it carries the `query`
|
|
145
|
+
* rather than a `url`, since search has no target URL.
|
|
146
|
+
*/
|
|
147
|
+
export class SearchFailedError extends MinifetchError {
|
|
148
|
+
query;
|
|
149
|
+
statusCode;
|
|
150
|
+
originalError;
|
|
151
|
+
/**
|
|
152
|
+
*
|
|
153
|
+
* @param query
|
|
154
|
+
* @param message
|
|
155
|
+
* @param statusCode
|
|
156
|
+
* @param originalError
|
|
157
|
+
*/
|
|
158
|
+
constructor(query, message, statusCode, originalError) {
|
|
159
|
+
super(message);
|
|
160
|
+
this.name = "SearchFailedError";
|
|
161
|
+
this.query = query;
|
|
162
|
+
this.statusCode = statusCode;
|
|
163
|
+
this.originalError = originalError;
|
|
164
|
+
Object.setPrototypeOf(this, SearchFailedError.prototype);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
@@ -15,7 +15,7 @@ import { PaymentFailedError, NetworkError } from "../types/errors.js";
|
|
|
15
15
|
* @param url
|
|
16
16
|
* @param config
|
|
17
17
|
*/
|
|
18
|
-
export async function handlePayment(url, config) {
|
|
18
|
+
export async function handlePayment(url, config, init) {
|
|
19
19
|
try {
|
|
20
20
|
const _x402Client = new x402Client();
|
|
21
21
|
let payer;
|
|
@@ -40,7 +40,8 @@ export async function handlePayment(url, config) {
|
|
|
40
40
|
throw new PaymentFailedError(`Unsupported network: ${config.network}`);
|
|
41
41
|
}
|
|
42
42
|
const fetchWithPayment = wrapFetchWithPayment(fetch, _x402Client);
|
|
43
|
-
|
|
43
|
+
// Default GET when no init passed; init carries method + JSON body for POST.
|
|
44
|
+
const response = await fetchWithPayment(url, init ?? { method: "GET" });
|
|
44
45
|
if (!response.ok) {
|
|
45
46
|
const serverMessage = await readServerErrorMessage(response);
|
|
46
47
|
throw new NetworkError(`Request failed: ${response.status} ${response.statusText}${serverMessage ? ` — ${serverMessage}` : ""}`);
|
|
@@ -72,10 +73,12 @@ export async function handlePayment(url, config) {
|
|
|
72
73
|
* @param url
|
|
73
74
|
* @param config
|
|
74
75
|
*/
|
|
75
|
-
export async function handleApiKeyRequest(url, config) {
|
|
76
|
+
export async function handleApiKeyRequest(url, config, init) {
|
|
76
77
|
const response = await fetch(url, {
|
|
77
|
-
|
|
78
|
+
...init,
|
|
79
|
+
method: init?.method ?? "GET",
|
|
78
80
|
headers: {
|
|
81
|
+
...init?.headers,
|
|
79
82
|
Authorization: `Bearer ${config.apiKey}`,
|
|
80
83
|
},
|
|
81
84
|
});
|
|
@@ -1,8 +1,13 @@
|
|
|
1
|
-
import { InvalidUrlError } from "../types/errors.js";
|
|
1
|
+
import { InvalidUrlError, InvalidQueryError } from "../types/errors.js";
|
|
2
2
|
/**
|
|
3
3
|
* Maximum allowed URL length
|
|
4
4
|
*/
|
|
5
5
|
const MAX_URL_LENGTH = 2048;
|
|
6
|
+
/**
|
|
7
|
+
* Maximum allowed search-query length. Mirrors Ceramic's hard cap on the server
|
|
8
|
+
* so we fail fast client-side instead of spending a paid call on a 400.
|
|
9
|
+
*/
|
|
10
|
+
const MAX_QUERY_LENGTH = 50;
|
|
6
11
|
/**
|
|
7
12
|
* Allowed URL protocols
|
|
8
13
|
*/
|
|
@@ -77,6 +82,27 @@ export function validateAndNormalizeUrl(url) {
|
|
|
77
82
|
}
|
|
78
83
|
return normalized;
|
|
79
84
|
}
|
|
85
|
+
/**
|
|
86
|
+
* Validate and normalize a keyword-search query.
|
|
87
|
+
* Mirrors the server: strips null bytes, normalizes unicode, trims, then enforces
|
|
88
|
+
* the same non-empty / 50-char rules so bad input fails before a paid call.
|
|
89
|
+
*
|
|
90
|
+
* @param query
|
|
91
|
+
* @throws {InvalidQueryError} if the query is empty or exceeds 50 characters
|
|
92
|
+
*/
|
|
93
|
+
export function validateAndNormalizeQuery(query) {
|
|
94
|
+
if (!query || typeof query !== "string") {
|
|
95
|
+
throw new InvalidQueryError(String(query ?? ""), "Query must be a non-empty string");
|
|
96
|
+
}
|
|
97
|
+
const cleaned = query.replace(/\0/g, "").normalize("NFC").trim();
|
|
98
|
+
if (!cleaned) {
|
|
99
|
+
throw new InvalidQueryError(query, "Query must not be empty");
|
|
100
|
+
}
|
|
101
|
+
if (cleaned.length > MAX_QUERY_LENGTH) {
|
|
102
|
+
throw new InvalidQueryError(cleaned, `Query exceeds maximum length of ${MAX_QUERY_LENGTH} characters`);
|
|
103
|
+
}
|
|
104
|
+
return cleaned;
|
|
105
|
+
}
|
|
80
106
|
/**
|
|
81
107
|
* Check if hostname is localhost or private IP
|
|
82
108
|
*
|
package/dist/types/client.d.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
import type { ClientConfig } from "./types/config.js";
|
|
2
|
-
import type { PreflightCheckResponse, PaidEndpointResponse } from "./types/responses.js";
|
|
1
|
+
import type { ClientConfig, HttpMethod } from "./types/config.js";
|
|
2
|
+
import type { PreflightCheckResponse, PaidEndpointResponse, SearchKeywordResponse } from "./types/responses.js";
|
|
3
3
|
/**
|
|
4
4
|
* Main Minifetch API client.
|
|
5
5
|
* Supports two auth modes:
|
|
6
6
|
* - x402: crypto micropayments via Coinbase x402 (pass network + privateKey)
|
|
7
7
|
* - apiKey: Stripe-backed credits (pass apiKey: "mf_prod_..." or "mf_dev_...")
|
|
8
|
+
*
|
|
9
|
+
* Every request method accepts an optional `method: "GET" | "POST"` in its
|
|
10
|
+
* options; it defaults to POST (params sent as a JSON body). Pass `method: "GET"`
|
|
11
|
+
* to send params in the query string instead.
|
|
8
12
|
*/
|
|
9
13
|
export declare class MinifetchClient {
|
|
10
14
|
private config;
|
|
@@ -13,28 +17,64 @@ export declare class MinifetchClient {
|
|
|
13
17
|
* @param config - Either { network, privateKey } for x402 or { apiKey } for API key auth
|
|
14
18
|
*/
|
|
15
19
|
constructor(config: ClientConfig);
|
|
20
|
+
/**
|
|
21
|
+
* Search the web by keyword (paid endpoint).
|
|
22
|
+
*
|
|
23
|
+
* Unlike the URL-based methods, this takes a search query rather than a URL and
|
|
24
|
+
* proxies to Minifetch's keyword search. Returns ranked results, each with a
|
|
25
|
+
* title, URL, and text snippet. `limit` and `descriptionLength` are convenience
|
|
26
|
+
* knobs the server clamps into range; the effective values (after clamping) are
|
|
27
|
+
* echoed back on `queryParameters`.
|
|
28
|
+
*
|
|
29
|
+
* @param query - keyword(s) to search for (1-50 characters)
|
|
30
|
+
* @param options
|
|
31
|
+
* @param options.limit - number of results, 1-10 (default 10). Out-of-range values are clamped.
|
|
32
|
+
* @param options.descriptionLength - max characters per snippet, 0-5000 (default 750, 0 = titles/URLs only). Out-of-range values are clamped.
|
|
33
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
34
|
+
* @throws {InvalidQueryError} if the query is empty or exceeds 50 characters
|
|
35
|
+
* @throws {SearchFailedError} if the search request fails
|
|
36
|
+
* @throws {PaymentFailedError} if x402 payment fails
|
|
37
|
+
* @throws {NetworkError} various reasons, check README
|
|
38
|
+
*/
|
|
39
|
+
searchByKeyword(query: string, options?: {
|
|
40
|
+
limit?: number;
|
|
41
|
+
descriptionLength?: number;
|
|
42
|
+
method?: HttpMethod;
|
|
43
|
+
}): Promise<SearchKeywordResponse>;
|
|
16
44
|
/**
|
|
17
45
|
* Check if URL is allowed by robots.txt (free preflight check — no auth required)
|
|
18
46
|
*
|
|
19
47
|
* @param url
|
|
20
48
|
* @param options
|
|
21
49
|
* @param options.fresh
|
|
50
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
22
51
|
* @throws {InvalidUrlError} if URL is invalid
|
|
23
52
|
* @throws {NetworkError} if request fails
|
|
24
53
|
*/
|
|
25
54
|
preflightUrlCheck(url: string, options?: {
|
|
26
55
|
fresh?: boolean;
|
|
56
|
+
method?: HttpMethod;
|
|
27
57
|
}): Promise<PreflightCheckResponse>;
|
|
28
58
|
/**
|
|
29
|
-
*
|
|
59
|
+
* INTERNAL / UNDOCUMENTED — deliberately omitted from the README and not part
|
|
60
|
+
* of the public API. The public {@link preflightUrlCheck} hits the FREE
|
|
61
|
+
* endpoint (no payment); this hits the PAID x402 twin at
|
|
62
|
+
* `/api/v1/x402/preflight/url-check` so our own suite can generate paid
|
|
63
|
+
* traffic against it — the x402 Bazaar weights usage for ranking and this
|
|
64
|
+
* refreshes the listing. x402 auth only; there is no session/api-key route
|
|
65
|
+
* for a paid url-check.
|
|
30
66
|
*
|
|
31
67
|
* @param url
|
|
32
|
-
* @
|
|
33
|
-
* @
|
|
34
|
-
* @
|
|
35
|
-
* @throws {
|
|
68
|
+
* @param options
|
|
69
|
+
* @param options.fresh - bypass the 24h robots.txt cache
|
|
70
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
71
|
+
* @throws {ConfigurationError} if the client is not in x402 mode
|
|
72
|
+
* @internal
|
|
36
73
|
*/
|
|
37
|
-
|
|
74
|
+
_exercisePaidUrlCheck(url: string, options?: {
|
|
75
|
+
fresh?: boolean;
|
|
76
|
+
method?: HttpMethod;
|
|
77
|
+
}): Promise<PaidEndpointResponse>;
|
|
38
78
|
/**
|
|
39
79
|
* Extract URL metadata (paid endpoint)
|
|
40
80
|
*
|
|
@@ -43,6 +83,7 @@ export declare class MinifetchClient {
|
|
|
43
83
|
* @param options.fields
|
|
44
84
|
* @param options.omitEmpty
|
|
45
85
|
* @param options.includeResponseBody
|
|
86
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
46
87
|
* @throws {InvalidUrlError} if URL is invalid
|
|
47
88
|
* @throws {ExtractionFailedError} various reasons, check README
|
|
48
89
|
* @throws {PaymentFailedError} if x402 payment fails
|
|
@@ -52,33 +93,43 @@ export declare class MinifetchClient {
|
|
|
52
93
|
fields?: string[];
|
|
53
94
|
omitEmpty?: boolean;
|
|
54
95
|
includeResponseBody?: boolean;
|
|
96
|
+
method?: HttpMethod;
|
|
55
97
|
}): Promise<PaidEndpointResponse>;
|
|
56
98
|
/**
|
|
57
99
|
* Extract URL links (paid endpoint)
|
|
58
100
|
*
|
|
59
101
|
* @param url
|
|
102
|
+
* @param options
|
|
103
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
60
104
|
* @throws {InvalidUrlError} if URL is invalid
|
|
61
105
|
* @throws {ExtractionFailedError} various reasons, check README
|
|
62
106
|
* @throws {PaymentFailedError} if x402 payment fails
|
|
63
107
|
* @throws {NetworkError} various reasons, check README
|
|
64
108
|
*/
|
|
65
|
-
extractUrlLinks(url: string
|
|
109
|
+
extractUrlLinks(url: string, options?: {
|
|
110
|
+
method?: HttpMethod;
|
|
111
|
+
}): Promise<PaidEndpointResponse>;
|
|
66
112
|
/**
|
|
67
113
|
* Extract URL preview (paid endpoint)
|
|
68
114
|
*
|
|
69
115
|
* @param url
|
|
116
|
+
* @param options
|
|
117
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
70
118
|
* @throws {InvalidUrlError} if URL is invalid
|
|
71
119
|
* @throws {ExtractionFailedError} various reasons, check README
|
|
72
120
|
* @throws {PaymentFailedError} if x402 payment fails
|
|
73
121
|
* @throws {NetworkError} various reasons, check README
|
|
74
122
|
*/
|
|
75
|
-
extractUrlPreview(url: string
|
|
123
|
+
extractUrlPreview(url: string, options?: {
|
|
124
|
+
method?: HttpMethod;
|
|
125
|
+
}): Promise<PaidEndpointResponse>;
|
|
76
126
|
/**
|
|
77
127
|
* Extract URL content as markdown (paid endpoint)
|
|
78
128
|
*
|
|
79
129
|
* @param url
|
|
80
130
|
* @param options
|
|
81
131
|
* @param options.includeMediaUrls
|
|
132
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
82
133
|
* @throws {InvalidUrlError} if URL is invalid
|
|
83
134
|
* @throws {ExtractionFailedError} various reasons, check README
|
|
84
135
|
* @throws {PaymentFailedError} if x402 payment fails
|
|
@@ -86,14 +137,33 @@ export declare class MinifetchClient {
|
|
|
86
137
|
*/
|
|
87
138
|
extractUrlContent(url: string, options?: {
|
|
88
139
|
includeMediaUrls?: boolean;
|
|
140
|
+
method?: HttpMethod;
|
|
141
|
+
}): Promise<PaidEndpointResponse>;
|
|
142
|
+
/**
|
|
143
|
+
* Run SEO page audit (paid endpoint)
|
|
144
|
+
*
|
|
145
|
+
* @param url
|
|
146
|
+
* @param options
|
|
147
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
148
|
+
* @throws {InvalidUrlError} if URL is invalid
|
|
149
|
+
* @throws {ExtractionFailedError} various reasons, check README
|
|
150
|
+
* @throws {PaymentFailedError} if x402 payment fails
|
|
151
|
+
* @throws {NetworkError} various reasons, check README
|
|
152
|
+
*/
|
|
153
|
+
runSeoPageAudit(url: string, options?: {
|
|
154
|
+
method?: HttpMethod;
|
|
89
155
|
}): Promise<PaidEndpointResponse>;
|
|
90
156
|
/**
|
|
91
157
|
* Check URL then run SEO page audit in one call.
|
|
92
158
|
* Throws RobotsBlockedError if robots.txt blocks the URL.
|
|
93
159
|
*
|
|
94
160
|
* @param url
|
|
161
|
+
* @param options
|
|
162
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
95
163
|
*/
|
|
96
|
-
checkAndRunSeoPageAudit(url: string
|
|
164
|
+
checkAndRunSeoPageAudit(url: string, options?: {
|
|
165
|
+
method?: HttpMethod;
|
|
166
|
+
}): Promise<PaidEndpointResponse>;
|
|
97
167
|
/**
|
|
98
168
|
* Check URL then extract metadata in one call.
|
|
99
169
|
* Throws RobotsBlockedError if robots.txt blocks the URL.
|
|
@@ -103,26 +173,36 @@ export declare class MinifetchClient {
|
|
|
103
173
|
* @param options.fields
|
|
104
174
|
* @param options.omitEmpty
|
|
105
175
|
* @param options.includeResponseBody
|
|
176
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
106
177
|
*/
|
|
107
178
|
checkAndExtractUrlMetadata(url: string, options?: {
|
|
108
179
|
fields?: string[];
|
|
109
180
|
omitEmpty?: boolean;
|
|
110
181
|
includeResponseBody?: boolean;
|
|
182
|
+
method?: HttpMethod;
|
|
111
183
|
}): Promise<PaidEndpointResponse>;
|
|
112
184
|
/**
|
|
113
185
|
* Check URL then extract links in one call.
|
|
114
186
|
* Throws RobotsBlockedError if robots.txt blocks the URL.
|
|
115
187
|
*
|
|
116
188
|
* @param url
|
|
189
|
+
* @param options
|
|
190
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
117
191
|
*/
|
|
118
|
-
checkAndExtractUrlLinks(url: string
|
|
192
|
+
checkAndExtractUrlLinks(url: string, options?: {
|
|
193
|
+
method?: HttpMethod;
|
|
194
|
+
}): Promise<PaidEndpointResponse>;
|
|
119
195
|
/**
|
|
120
196
|
* Check URL then extract preview in one call.
|
|
121
197
|
* Throws RobotsBlockedError if robots.txt blocks the URL.
|
|
122
198
|
*
|
|
123
199
|
* @param url
|
|
200
|
+
* @param options
|
|
201
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
124
202
|
*/
|
|
125
|
-
checkAndExtractUrlPreview(url: string
|
|
203
|
+
checkAndExtractUrlPreview(url: string, options?: {
|
|
204
|
+
method?: HttpMethod;
|
|
205
|
+
}): Promise<PaidEndpointResponse>;
|
|
126
206
|
/**
|
|
127
207
|
* Check URL then extract content in one call.
|
|
128
208
|
* Throws RobotsBlockedError if robots.txt blocks the URL.
|
|
@@ -130,9 +210,11 @@ export declare class MinifetchClient {
|
|
|
130
210
|
* @param url
|
|
131
211
|
* @param options
|
|
132
212
|
* @param options.includeMediaUrls
|
|
213
|
+
* @param options.method - "GET" or "POST" (default POST)
|
|
133
214
|
*/
|
|
134
215
|
checkAndExtractUrlContent(url: string, options?: {
|
|
135
216
|
includeMediaUrls?: boolean;
|
|
217
|
+
method?: HttpMethod;
|
|
136
218
|
}): Promise<PaidEndpointResponse>;
|
|
137
219
|
/**
|
|
138
220
|
* Returns the correct paid path segment based on auth mode.
|
|
@@ -143,15 +225,38 @@ export declare class MinifetchClient {
|
|
|
143
225
|
*/
|
|
144
226
|
private _paidPath;
|
|
145
227
|
/**
|
|
146
|
-
*
|
|
147
|
-
*
|
|
228
|
+
* Encode a request for the wire. GET → params in the query string, no body.
|
|
229
|
+
* POST → params as a JSON body with a Content-Type header. Auth headers
|
|
230
|
+
* (Bearer / x402 payment) are added downstream, not here.
|
|
231
|
+
*
|
|
232
|
+
* @param path - absolute API path (already includes /api/v1[/x402])
|
|
233
|
+
* @param params - request params (string or boolean values)
|
|
234
|
+
* @param method - "GET" or "POST"
|
|
235
|
+
* @returns the full request URL and the fetch init (method + optional body/headers)
|
|
236
|
+
*/
|
|
237
|
+
private _buildRequest;
|
|
238
|
+
/**
|
|
239
|
+
* Build the request, dispatch to the correct auth handler, then normalize the
|
|
240
|
+
* response into PaidEndpointResponse.
|
|
148
241
|
* Note: payment field is only present for x402 responses.
|
|
149
242
|
*
|
|
150
|
-
* @param
|
|
243
|
+
* @param endpoint - endpoint path segment, e.g. "/extract/url-metadata"
|
|
151
244
|
* @param normalizedUrl
|
|
152
245
|
* @param label - used in error messages
|
|
246
|
+
* @param params - request params (string or boolean values)
|
|
247
|
+
* @param method - "GET" or "POST" (default POST)
|
|
153
248
|
*/
|
|
154
249
|
private _makeRequest;
|
|
250
|
+
/**
|
|
251
|
+
* Search-specific request path. Mirrors {@link _makeRequest} but for the
|
|
252
|
+
* keyword search endpoint: there is no URL, non-OK responses surface as
|
|
253
|
+
* SearchFailedError (carrying the query), and the server's clamped
|
|
254
|
+
* `queryParameters` echo is preserved rather than dropped.
|
|
255
|
+
*
|
|
256
|
+
* @param params - request params (query + optional limit/descriptionLength)
|
|
257
|
+
* @param method - "GET" or "POST" (default POST)
|
|
258
|
+
*/
|
|
259
|
+
private _makeSearchRequest;
|
|
155
260
|
/**
|
|
156
261
|
* Preflight check helper — throws RobotsBlockedError if not allowed
|
|
157
262
|
*
|
|
@@ -166,4 +271,13 @@ export declare class MinifetchClient {
|
|
|
166
271
|
* @param label
|
|
167
272
|
*/
|
|
168
273
|
private _rethrowError;
|
|
274
|
+
/**
|
|
275
|
+
* Search sibling of {@link _rethrowError}: re-throw known search error types,
|
|
276
|
+
* wrapping anything else in SearchFailedError (which carries the query, not a URL).
|
|
277
|
+
*
|
|
278
|
+
* @param error
|
|
279
|
+
* @param query
|
|
280
|
+
* @param label
|
|
281
|
+
*/
|
|
282
|
+
private _rethrowSearchError;
|
|
169
283
|
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
export declare const VALID_NETWORKS: readonly ["base", "base-sepolia", "solana", "solana-devnet"];
|
|
2
2
|
export type Network = (typeof VALID_NETWORKS)[number];
|
|
3
|
+
/** HTTP method for a request. Every endpoint accepts GET (query string) or POST (JSON body). */
|
|
4
|
+
export type HttpMethod = "GET" | "POST";
|
|
3
5
|
/**
|
|
4
6
|
* Config for x402 crypto micropayment auth
|
|
5
7
|
*/
|
|
@@ -84,3 +84,35 @@ export declare class NetworkError extends MinifetchError {
|
|
|
84
84
|
*/
|
|
85
85
|
constructor(message: string, originalError?: Error);
|
|
86
86
|
}
|
|
87
|
+
/**
|
|
88
|
+
* Thrown when keyword-search query validation fails (empty, or over 50 chars).
|
|
89
|
+
* The query sibling of {@link InvalidUrlError} — searchByKeyword takes a query,
|
|
90
|
+
* not a URL, so bad input surfaces here instead.
|
|
91
|
+
*/
|
|
92
|
+
export declare class InvalidQueryError extends MinifetchError {
|
|
93
|
+
readonly query: string;
|
|
94
|
+
/**
|
|
95
|
+
*
|
|
96
|
+
* @param query
|
|
97
|
+
* @param message
|
|
98
|
+
*/
|
|
99
|
+
constructor(query: string, message?: string);
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Thrown when a keyword search fails (non-OK response or unexpected error).
|
|
103
|
+
* The search sibling of {@link ExtractionFailedError}: it carries the `query`
|
|
104
|
+
* rather than a `url`, since search has no target URL.
|
|
105
|
+
*/
|
|
106
|
+
export declare class SearchFailedError extends MinifetchError {
|
|
107
|
+
readonly query: string;
|
|
108
|
+
readonly statusCode?: number;
|
|
109
|
+
readonly originalError?: Error;
|
|
110
|
+
/**
|
|
111
|
+
*
|
|
112
|
+
* @param query
|
|
113
|
+
* @param message
|
|
114
|
+
* @param statusCode
|
|
115
|
+
* @param originalError
|
|
116
|
+
*/
|
|
117
|
+
constructor(query: string, message: string, statusCode?: number, originalError?: Error);
|
|
118
|
+
}
|
|
@@ -48,3 +48,38 @@ export interface PaymentInfo {
|
|
|
48
48
|
/** Link to view transaction on block explorer **/
|
|
49
49
|
explorerLink: string;
|
|
50
50
|
}
|
|
51
|
+
/**
|
|
52
|
+
* A single keyword-search result.
|
|
53
|
+
*/
|
|
54
|
+
export interface SearchKeywordResult {
|
|
55
|
+
/** Title of the result page */
|
|
56
|
+
title: string;
|
|
57
|
+
/** URL of the result page */
|
|
58
|
+
url: string;
|
|
59
|
+
/** Text snippet from the result page, trimmed to the requested descriptionLength */
|
|
60
|
+
description: string;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Response from the keyword search endpoint.
|
|
64
|
+
*
|
|
65
|
+
* Unlike PaidEndpointResponse, this preserves `queryParameters` — the effective
|
|
66
|
+
* request params after the server clamps `limit`/`descriptionLength` into range.
|
|
67
|
+
* That echo is how a caller learns what actually ran (no black box).
|
|
68
|
+
*/
|
|
69
|
+
export interface SearchKeywordResponse {
|
|
70
|
+
/** Minifetch API success (200, ok) */
|
|
71
|
+
success: boolean;
|
|
72
|
+
/** Effective params after server-side clamping. */
|
|
73
|
+
queryParameters: {
|
|
74
|
+
query: string;
|
|
75
|
+
limit: number;
|
|
76
|
+
descriptionLength: number;
|
|
77
|
+
};
|
|
78
|
+
/** Ranked search results. */
|
|
79
|
+
results: Array<{
|
|
80
|
+
data: SearchKeywordResult;
|
|
81
|
+
error?: Record<string, any>;
|
|
82
|
+
}>;
|
|
83
|
+
/** Payment information — only present for paid x402 requests. */
|
|
84
|
+
payment?: PaymentInfo;
|
|
85
|
+
}
|
|
@@ -10,7 +10,7 @@ import type { PaymentInfo } from "../types/responses.js";
|
|
|
10
10
|
* @param url
|
|
11
11
|
* @param config
|
|
12
12
|
*/
|
|
13
|
-
export declare function handlePayment(url: string, config: InitializedConfig): Promise<{
|
|
13
|
+
export declare function handlePayment(url: string, config: InitializedConfig, init?: RequestInit): Promise<{
|
|
14
14
|
response: Response;
|
|
15
15
|
payment?: PaymentInfo;
|
|
16
16
|
}>;
|
|
@@ -21,6 +21,6 @@ export declare function handlePayment(url: string, config: InitializedConfig): P
|
|
|
21
21
|
* @param url
|
|
22
22
|
* @param config
|
|
23
23
|
*/
|
|
24
|
-
export declare function handleApiKeyRequest(url: string, config: InitializedConfig): Promise<{
|
|
24
|
+
export declare function handleApiKeyRequest(url: string, config: InitializedConfig, init?: RequestInit): Promise<{
|
|
25
25
|
response: Response;
|
|
26
26
|
}>;
|
|
@@ -6,3 +6,12 @@
|
|
|
6
6
|
* @throws {InvalidUrlError} if URL is invalid
|
|
7
7
|
*/
|
|
8
8
|
export declare function validateAndNormalizeUrl(url: string): string;
|
|
9
|
+
/**
|
|
10
|
+
* Validate and normalize a keyword-search query.
|
|
11
|
+
* Mirrors the server: strips null bytes, normalizes unicode, trims, then enforces
|
|
12
|
+
* the same non-empty / 50-char rules so bad input fails before a paid call.
|
|
13
|
+
*
|
|
14
|
+
* @param query
|
|
15
|
+
* @throws {InvalidQueryError} if the query is empty or exceeds 50 characters
|
|
16
|
+
*/
|
|
17
|
+
export declare function validateAndNormalizeQuery(query: string): string;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "minifetch-api",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "Minifetch.com API Client.
|
|
3
|
+
"version": "1.7.0",
|
|
4
|
+
"description": "Minifetch.com API Client. Search, scrape, extract and SEO audit web pages. Pay per fetch, no subscription.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/esm/index.js",
|
|
7
7
|
"types": "./dist/types/index.d.ts",
|
|
@@ -25,6 +25,8 @@
|
|
|
25
25
|
"metadata extraction",
|
|
26
26
|
"content extraction",
|
|
27
27
|
"link extraction",
|
|
28
|
+
"search",
|
|
29
|
+
"keyword search",
|
|
28
30
|
"hosted",
|
|
29
31
|
"meta tags",
|
|
30
32
|
"open graph",
|