@xyz17711/web-scraper 0.0.0-stage → 0.1.1

package/README.md

@@ -1,3 +1,90 @@
-# Temporary Holding Version
+# @xyz17711/web-scraper
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+[![Publish](https://github.com/17711-services/clients/actions/workflows/publish-node.yml/badge.svg)](https://github.com/17711-services/clients/actions/workflows/publish-node.yml)
+[![npm](https://img.shields.io/npm/v/@xyz17711/web-scraper)](https://www.npmjs.com/package/@xyz17711/web-scraper)
+[![License](https://img.shields.io/npm/l/@xyz17711/web-scraper)](https://github.com/17711-services/clients/blob/main/LICENSE)
+
+Typed TypeScript client for **web-scraper**, part of the
+[17711.xyz](https://api.17711.xyz) microservice fleet -- pay-per-call APIs settled in USDC via
+[x402](https://x402.org), listed on [x402scan](https://www.x402scan.com/server/f1c7b972-3405-4b75-8b4f-d1bfbd6305ff).
+
+## Install
+
+```bash
+npm install @xyz17711/web-scraper
+```
+
+## Methods
+
+| Method | Description |
+| --- | --- |
+| `extractMarkdown` | URL to Markdown: read any web page and get its main article content as clean Markdown for agents ($0.003, x402) |
+| `getTaskStatus` | Polls for the result of a task enqueued by a *_async tool |
+| `scrapeUrl` | Scrape URL: fetch any web page through a proxy and get its status, content type and raw HTML ($0.002, x402) |
+| `scrapeUrlAsync` | [Deprecated - use web_scraper_scrape_url] Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.002, x402) |
+
+## Quickstart
+
+```typescript
+import { Client } from "@xyz17711/web-scraper";
+
+const client = new Client("https://api.17711.xyz/registry/web-scraper");
+// see the Methods table above for every available call and its request/response type
+```
+
+## Paying past the free quota
+
+Every gated tool allows a small number of free calls per day; beyond that it returns a 402
+challenge. Without a wallet configured, this client throws `PaymentRequiredError` with the price,
+network, and payee address attached:
+
+```typescript
+import { Client, PaymentRequiredError } from "@xyz17711/web-scraper";
+
+const client = new Client("https://api.17711.xyz/registry/web-scraper");
+try {
+  await client.extractMarkdown(/* ... */);
+} catch (e) {
+  if (e instanceof PaymentRequiredError) {
+    console.log(`needs ${e.priceUsdc} USDC on ${e.network}, payable to ${e.payTo}`);
+  }
+}
+```
+
+To pay automatically, configure a wallet on the client -- the private key should come from your
+own environment/secrets manager, never a literal string in source code:
+
+```typescript
+import { Client } from "@xyz17711/web-scraper";
+
+const client = new Client("https://api.17711.xyz/registry/web-scraper", {
+  x402PrivateKey: process.env.X402_PRIVATE_KEY,
+  x402ExpectedNetwork: "eip155:8453", // Base mainnet; use eip155:84532 for Base Sepolia testnet
+  x402MaxPaymentUsdc: 0.01,           // refuse any single call above this
+  x402MaxTotalSpendUsdc: 1.0,         // refuse once this WALLET's lifetime spend would exceed it
+});
+```
+
+**Never commit a private key.** Both spend caps are required together with the key -- a wallet
+with real funds and no explicit caps refuses to construct at all (`WalletConfigError`), on
+purpose. The cumulative cap is tracked per wallet address, shared by every `Client` you construct
+with that same key in this process.
+
+## LangChain.js
+
+```typescript
+import { Client } from "@xyz17711/web-scraper";
+import { WebScraperToolkit } from "@xyz17711/web-scraper/langchain";
+
+const client = new Client("https://api.17711.xyz/registry/web-scraper", { /* ...wallet options... */ });
+const tools = new WebScraperToolkit(client).getTools();
+```
+
+Requires the optional `@langchain/core`+`zod` peer dependencies (`npm install @langchain/core
+zod`) -- importing the base package above never requires them; only `@xyz17711/web-scraper/langchain`
+does.
+
+## License
+
+MIT -- see [`LICENSE`](../../LICENSE), shared by every language's client library in the
+`clients` repo this package lives in.

package/dist/index.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Typed HTTP client and request/response interfaces for web-scraper.
+ */
+import { Client as BaseClient, type ClientOptions } from "@xyz17711/core";
+export interface ExtractMarkdownResponse {
+    content_type?: string | null;
+    /** The URL actually reached, after any redirects the target sent. */
+    final_url: string;
+    /** Boilerplate-stripped main article content, as Markdown -- navigation, ads, and sidebars removed, not the raw page HTML. */
+    markdown: string;
+    /** The REAL HTTP status the underlying fetch received. */
+    status: number;
+    /** Article title, if Readability found one. */
+    title?: string | null;
+    /** `true` if the underlying fetch's HTML was truncated by web-scraper's own fixed byte ceiling (see `web_scraper_scrape_url`'s own `truncated` field) -- `markdown` may then also be incomplete. */
+    truncated: boolean;
+    /** The URL as requested, verbatim. */
+    url: string;
+}
+export interface ExtractMarkdownRequest {
+    /** The target URL to fetch, e.g. `"https://example.com/article"`. Must be `http://` or `https://` -- same validation as `web_scraper_scrape_url`, since this tool fetches through the exact same internal scraper. */
+    url: string;
+}
+/** `"pending"` (enqueued, no result yet), `"finished"` (a worker wrote a result -- see `ok`/ `result`), or `"unknown"` (this id was never valid, or both it and its result have expired -- see [`nimpoleon_tasks::TaskState`]'s own doc comment for why this is 3 states, not 2). */
+export interface TaskStatusKind {
+}
+export interface GetTaskStatusResponse {
+    ok?: boolean | null;
+    result?: unknown;
+    status: TaskStatusKind;
+}
+export interface GetTaskStatusRequest {
+    /** The `task_id` a `*_async` tool returned. */
+    task_id: string;
+}
+export interface ScrapeUrlResponse {
+    content_type?: string | null;
+    /** The URL actually reached, after any redirects the target sent. */
+    final_url: string;
+    /** Raw response body, decoded as UTF-8 (lossy -- see this tool's own description for why), truncated to this service's own fixed byte ceiling if needed. */
+    html: string;
+    /** The REAL HTTP status the target returned -- never masked, never itself a reason to retry. */
+    status: number;
+    /** `true` if `html` was cut short by this service's own fixed byte ceiling. */
+    truncated: boolean;
+    /** The URL as requested, verbatim. */
+    url: string;
+}
+export interface ScrapeUrlRequest {
+    /** The target URL to fetch, e.g. `"https://example.com/page"`. Must be `http://` or `https://` -- any other scheme is rejected before any fetch is attempted. */
+    url: string;
+}
+/** The fixed response shape of every `*_async` tool -- deliberately NOT generic over the underlying sync tool's own response type, since no work has actually happened yet at this point. */
+export interface AsyncTaskResponse {
+    task_id: string;
+}
+export interface ScrapeUrlAsyncRequest {
+    /** When set, the worker that processes this task POSTs the eventual result (a `TaskResult` JSON body) here, best-effort -- polling `web_scraper_get_task_status` always works too, regardless of whether this is set. */
+    callback_url?: string | null;
+    /** The target URL to fetch, e.g. `"https://example.com/page"`. Must be `http://` or `https://` -- any other scheme is rejected before any fetch is attempted. */
+    url: string;
+}
+/** Typed client for web-scraper, e.g. `client.extractMarkdown(...)`. */
+export declare class Client extends BaseClient {
+    /** URL to Markdown: read any web page and get its main article content as clean Markdown for agents ($0.003, x402) */
+    extractMarkdown(request: ExtractMarkdownRequest): Promise<ExtractMarkdownResponse>;
+    /** Polls for the result of a task enqueued by a *_async tool */
+    getTaskStatus(request: GetTaskStatusRequest): Promise<GetTaskStatusResponse>;
+    /** Scrape URL: fetch any web page through a proxy and get its status, content type and raw HTML ($0.002, x402) */
+    scrapeUrl(request: ScrapeUrlRequest): Promise<ScrapeUrlResponse>;
+    /** [Deprecated - use web_scraper_scrape_url] Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.002, x402) */
+    scrapeUrlAsync(request: ScrapeUrlAsyncRequest): Promise<AsyncTaskResponse>;
+}
+export { ClientOptions };

package/dist/index.js

@@ -0,0 +1,23 @@
+/**
+ * Typed HTTP client and request/response interfaces for web-scraper.
+ */
+import { Client as BaseClient } from "@xyz17711/core";
+/** Typed client for web-scraper, e.g. `client.extractMarkdown(...)`. */
+export class Client extends BaseClient {
+    /** URL to Markdown: read any web page and get its main article content as clean Markdown for agents ($0.003, x402) */
+    async extractMarkdown(request) {
+        return this.post("/v1/web-scraper/extract_markdown", request);
+    }
+    /** Polls for the result of a task enqueued by a *_async tool */
+    async getTaskStatus(request) {
+        return this.post("/v1/web-scraper/get_task_status", request);
+    }
+    /** Scrape URL: fetch any web page through a proxy and get its status, content type and raw HTML ($0.002, x402) */
+    async scrapeUrl(request) {
+        return this.post("/v1/web-scraper/scrape_url", request);
+    }
+    /** [Deprecated - use web_scraper_scrape_url] Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.002, x402) */
+    async scrapeUrlAsync(request) {
+        return this.post("/v1/web-scraper/scrape_url_async", request);
+    }
+}

package/dist/langchain.d.ts

@@ -0,0 +1,37 @@
+/**
+ * LangChain.js tools for web-scraper, grouped in one Toolkit -- one tool per operation, each just an async function (JS has no separate sync form to generate). Requires the `@langchain/core`+`zod` peer dependencies declared in this package's own package.json -- imported from `web-scraper/langchain`, never from the package root, so importing the base client never requires them installed.
+ */
+import { z } from "zod";
+import { Client } from "./index.js";
+export declare class WebScraperToolkit {
+    private client;
+    constructor(client: Client);
+    getTools(): (import("@langchain/core/tools").DynamicStructuredTool<z.ZodObject<{
+        url: z.ZodString;
+    }, z.core.$strip>, {
+        url: string;
+    }, {
+        url: string;
+    }, import("./index.js").ExtractMarkdownResponse, unknown, "extractMarkdown"> | import("@langchain/core/tools").DynamicStructuredTool<z.ZodObject<{
+        task_id: z.ZodString;
+    }, z.core.$strip>, {
+        task_id: string;
+    }, {
+        task_id: string;
+    }, import("./index.js").GetTaskStatusResponse, unknown, "getTaskStatus"> | import("@langchain/core/tools").DynamicStructuredTool<z.ZodObject<{
+        url: z.ZodString;
+    }, z.core.$strip>, {
+        url: string;
+    }, {
+        url: string;
+    }, import("./index.js").ScrapeUrlResponse, unknown, "scrapeUrl"> | import("@langchain/core/tools").DynamicStructuredTool<z.ZodObject<{
+        callback_url: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+        url: z.ZodString;
+    }, z.core.$strip>, {
+        url: string;
+        callback_url?: string | null | undefined;
+    }, {
+        url: string;
+        callback_url?: string | null | undefined;
+    }, import("./index.js").AsyncTaskResponse, unknown, "scrapeUrlAsync">)[];
+}

package/dist/langchain.js

@@ -0,0 +1,34 @@
+/**
+ * LangChain.js tools for web-scraper, grouped in one Toolkit -- one tool per operation, each just an async function (JS has no separate sync form to generate). Requires the `@langchain/core`+`zod` peer dependencies declared in this package's own package.json -- imported from `web-scraper/langchain`, never from the package root, so importing the base client never requires them installed.
+ */
+import { tool } from "@langchain/core/tools";
+import { z } from "zod";
+export class WebScraperToolkit {
+    constructor(client) {
+        this.client = client;
+    }
+    getTools() {
+        return [
+            tool(async (input) => this.client.extractMarkdown(input), {
+                name: "extractMarkdown",
+                description: "URL to Markdown: read any web page and get its main article content as clean Markdown for agents ($0.003, x402)",
+                schema: z.object({ url: z.string() }),
+            }),
+            tool(async (input) => this.client.getTaskStatus(input), {
+                name: "getTaskStatus",
+                description: "Polls for the result of a task enqueued by a *_async tool",
+                schema: z.object({ task_id: z.string() }),
+            }),
+            tool(async (input) => this.client.scrapeUrl(input), {
+                name: "scrapeUrl",
+                description: "Scrape URL: fetch any web page through a proxy and get its status, content type and raw HTML ($0.002, x402)",
+                schema: z.object({ url: z.string() }),
+            }),
+            tool(async (input) => this.client.scrapeUrlAsync(input), {
+                name: "scrapeUrlAsync",
+                description: "[Deprecated - use web_scraper_scrape_url] Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.002, x402)",
+                schema: z.object({ callback_url: z.string().nullable().optional(), url: z.string() }),
+            }),
+        ];
+    }
+}

package/package.json

@@ -1,6 +1,70 @@
 {
   "name": "@xyz17711/web-scraper",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "description": "Typed TypeScript client for the 17711.xyz Web Scraper API: pay-per-call, settled in USDC via x402.",
+  "license": "MIT",
+  "author": "17711 Services",
+  "homepage": "https://api.17711.xyz",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/17711-services/clients.git",
+    "directory": "node/packages/web-scraper"
+  },
+  "bugs": {
+    "url": "https://github.com/17711-services/clients/issues"
+  },
+  "keywords": [
+    "x402",
+    "usdc",
+    "api-client",
+    "pay-per-call",
+    "ai-agents",
+    "mcp",
+    "langchain",
+    "web-scraper"
+  ],
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./langchain": {
+      "types": "./dist/langchain.d.ts",
+      "default": "./dist/langchain.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc -p .",
+    "test": "npm run build && node --test test/*.test.mjs"
+  },
+  "dependencies": {
+    "@xyz17711/core": "^0.1.0"
+  },
+  "peerDependencies": {
+    "@langchain/core": ">=0.3",
+    "zod": "^3 || ^4"
+  },
+  "peerDependenciesMeta": {
+    "@langchain/core": {
+      "optional": true
+    },
+    "zod": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "typescript": "^5",
+    "@langchain/core": "^1.2",
+    "zod": "^4"
+  }
+}