@xyz17711/web-scraper 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -4
- package/dist/index.d.ts +9 -0
- package/dist/index.js +8 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -28,10 +28,13 @@ npm install @xyz17711/web-scraper
|
|
|
28
28
|
```typescript
|
|
29
29
|
import { Client } from "@xyz17711/web-scraper";
|
|
30
30
|
|
|
31
|
-
const client = new Client(
|
|
31
|
+
const client = new Client(); // talks to https://api.17711.xyz/registry/web-scraper by default
|
|
32
32
|
// see the Methods table above for every available call and its request/response type
|
|
33
33
|
```
|
|
34
34
|
|
|
35
|
+
Pass another base URL (self-hosted instance, test server) as the first argument:
|
|
36
|
+
`new Client("https://my-host.example/registry/web-scraper")`.
|
|
37
|
+
|
|
35
38
|
## Paying past the free quota
|
|
36
39
|
|
|
37
40
|
Every gated tool allows a small number of free calls per day; beyond that it returns a 402
|
|
@@ -41,7 +44,7 @@ network, and payee address attached:
|
|
|
41
44
|
```typescript
|
|
42
45
|
import { Client, PaymentRequiredError } from "@xyz17711/web-scraper";
|
|
43
46
|
|
|
44
|
-
const client = new Client(
|
|
47
|
+
const client = new Client();
|
|
45
48
|
try {
|
|
46
49
|
await client.extractMarkdown(/* ... */);
|
|
47
50
|
} catch (e) {
|
|
@@ -57,7 +60,7 @@ own environment/secrets manager, never a literal string in source code:
|
|
|
57
60
|
```typescript
|
|
58
61
|
import { Client } from "@xyz17711/web-scraper";
|
|
59
62
|
|
|
60
|
-
const client = new Client(
|
|
63
|
+
const client = new Client({
|
|
61
64
|
x402PrivateKey: process.env.X402_PRIVATE_KEY,
|
|
62
65
|
x402ExpectedNetwork: "eip155:8453", // Base mainnet; use eip155:84532 for Base Sepolia testnet
|
|
63
66
|
x402MaxPaymentUsdc: 0.01, // refuse any single call above this
|
|
@@ -76,7 +79,7 @@ with that same key in this process.
|
|
|
76
79
|
import { Client } from "@xyz17711/web-scraper";
|
|
77
80
|
import { WebScraperToolkit } from "@xyz17711/web-scraper/langchain";
|
|
78
81
|
|
|
79
|
-
const client = new Client(
|
|
82
|
+
const client = new Client({ /* ...wallet options... */ });
|
|
80
83
|
const tools = new WebScraperToolkit(client).getTools();
|
|
81
84
|
```
|
|
82
85
|
|
package/dist/index.d.ts
CHANGED
|
@@ -60,8 +60,17 @@ export interface ScrapeUrlAsyncRequest {
|
|
|
60
60
|
/** The target URL to fetch, e.g. `"https://example.com/page"`. Must be `http://` or `https://` -- any other scheme is rejected before any fetch is attempted. */
|
|
61
61
|
url: string;
|
|
62
62
|
}
|
|
63
|
+
/** Production base URL of this service; pass another to self-host or test. */
|
|
64
|
+
export declare const DEFAULT_BASE_URL = "https://api.17711.xyz/registry/web-scraper";
|
|
63
65
|
/** Typed client for web-scraper, e.g. `client.extractMarkdown(...)`. */
|
|
64
66
|
export declare class Client extends BaseClient {
|
|
67
|
+
/**
|
|
68
|
+
* `new Client()` talks to the production service. Pass a base URL to target another
|
|
69
|
+
* instance, and/or wallet options (`new Client({ x402PrivateKey, ... })`) to pay past the
|
|
70
|
+
* free quota automatically.
|
|
71
|
+
*/
|
|
72
|
+
constructor(options?: ClientOptions);
|
|
73
|
+
constructor(baseUrl?: string, options?: ClientOptions);
|
|
65
74
|
/** URL to Markdown: read any web page and get its main article content as clean Markdown for agents ($0.003, x402) */
|
|
66
75
|
extractMarkdown(request: ExtractMarkdownRequest): Promise<ExtractMarkdownResponse>;
|
|
67
76
|
/** Polls for the result of a task enqueued by a *_async tool */
|
package/dist/index.js
CHANGED
|
@@ -2,8 +2,16 @@
|
|
|
2
2
|
* Typed HTTP client and request/response interfaces for web-scraper.
|
|
3
3
|
*/
|
|
4
4
|
import { Client as BaseClient } from "@xyz17711/core";
|
|
5
|
+
/** Production base URL of this service; pass another to self-host or test. */
|
|
6
|
+
export const DEFAULT_BASE_URL = "https://api.17711.xyz/registry/web-scraper";
|
|
5
7
|
/** Typed client for web-scraper, e.g. `client.extractMarkdown(...)`. */
|
|
6
8
|
export class Client extends BaseClient {
|
|
9
|
+
constructor(baseUrlOrOptions, options = {}) {
|
|
10
|
+
const [url, opts] = typeof baseUrlOrOptions === "object"
|
|
11
|
+
? [DEFAULT_BASE_URL, baseUrlOrOptions]
|
|
12
|
+
: [baseUrlOrOptions ?? DEFAULT_BASE_URL, options];
|
|
13
|
+
super(url, opts);
|
|
14
|
+
}
|
|
7
15
|
/** URL to Markdown: read any web page and get its main article content as clean Markdown for agents ($0.003, x402) */
|
|
8
16
|
async extractMarkdown(request) {
|
|
9
17
|
return this.post("/v1/web-scraper/extract_markdown", request);
|
package/package.json
CHANGED